From 8e727588589818a1c7ff9072245b28d6ac2b48f3 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sat, 18 Nov 2023 13:16:33 +0530 Subject: [PATCH 001/235] Introduced internal packaged, added resource which now replaces Component --- api/v1alpha1/types_etcd.go | 95 +- internal/client/kubernetes/types.go | 35 + internal/common/constants.go | 48 + internal/controller/compaction/config.go | 95 ++ internal/controller/compaction/metrics.go | 112 +++ internal/controller/compaction/reconciler.go | 558 ++++++++++ internal/controller/compaction/register.go | 45 + internal/controller/config.go | 172 ++++ internal/controller/etcd/config.go | 92 ++ internal/controller/etcd/reconcile_delete.go | 115 +++ internal/controller/etcd/reconcile_spec.go | 71 ++ internal/controller/etcd/reconciler.go | 183 ++++ internal/controller/etcd/register.go | 31 + .../controller/etcdcopybackupstask/config.go | 63 ++ .../etcdcopybackupstask_suite_test.go | 31 + .../etcdcopybackupstask/reconciler.go | 557 ++++++++++ .../etcdcopybackupstask/reconciler_test.go | 950 ++++++++++++++++++ .../etcdcopybackupstask/register.go | 42 + internal/controller/manager.go | 130 +++ internal/controller/predicate/predicate.go | 221 ++++ .../predicate/predicate_suite_test.go | 27 + .../controller/predicate/predicate_test.go | 668 ++++++++++++ internal/controller/secret/config.go | 44 + internal/controller/secret/reconciler.go | 116 +++ internal/controller/secret/reconciler_test.go | 240 +++++ internal/controller/secret/register.go | 50 + .../controller/secret/secret_suite_test.go | 31 + internal/controller/utils/etcdstatus.go | 91 ++ internal/controller/utils/reconciler.go | 116 +++ internal/controller/utils/utils_suite_test.go | 27 + internal/controller/utils/validator.go | 50 + internal/controller/utils/validator_test.go | 53 + internal/errors/errors.go | 49 + internal/features/features.go | 44 + internal/health/condition/builder.go | 141 +++ internal/health/condition/builder_test.go | 187 ++++ .../health/condition/check_all_members.go | 69 ++ .../condition/check_all_members_test.go | 124 +++ .../health/condition/check_backup_ready.go | 142 +++ .../condition/check_backup_ready_test.go | 275 +++++ internal/health/condition/check_ready.go | 71 ++ internal/health/condition/check_ready_test.go | 135 +++ .../health/condition/condition_suite_test.go | 27 + internal/health/condition/types.go | 57 ++ internal/health/etcdmember/builder.go | 115 +++ internal/health/etcdmember/builder_test.go | 187 ++++ internal/health/etcdmember/check_ready.go | 157 +++ .../health/etcdmember/check_ready_test.go | 403 ++++++++ .../etcdmember/etcdmember_suite_test.go | 27 + internal/health/etcdmember/types.go | 63 ++ internal/health/status/check.go | 149 +++ internal/health/status/check_test.go | 282 ++++++ internal/health/status/status_suite_test.go | 27 + internal/mapper/etcd_to_secret.go | 82 ++ internal/mapper/etcd_to_secret_test.go | 115 +++ internal/mapper/mapper_suite_test.go | 27 + internal/mapper/statefulset_to_etcd.go | 76 ++ internal/mapper/statefulset_to_etcd_test.go | 125 +++ internal/metrics/metrics.go | 120 +++ internal/metrics/metrics_suite_test.go | 27 + internal/metrics/metrics_test.go | 90 ++ .../mock/controller-runtime/client/doc.go | 16 + .../mock/controller-runtime/client/mocks.go | 487 +++++++++ .../resource/clientservice/clientservice.go | 144 +++ internal/resource/configmap/configmap.go | 97 ++ internal/resource/configmap/etcdconfig.go | 125 +++ internal/resource/memberlease/memberlease.go | 105 ++ internal/resource/peerservice/peerservice.go | 95 ++ .../poddisruptionbudget.go | 91 ++ internal/resource/resource.go | 105 ++ internal/resource/role/role.go | 85 ++ internal/resource/rolebinding/rolebinding.go | 88 ++ .../resource/serviceaccount/serviceaccount.go | 70 ++ .../resource/snapshotlease/snapshotlease.go | 128 +++ internal/utils/concurrent.go | 77 ++ internal/utils/image.go | 83 ++ internal/utils/image_test.go | 172 ++++ internal/utils/lease.go | 74 ++ internal/utils/miscellaneous.go | 102 ++ internal/utils/statefulset.go | 63 ++ internal/utils/statefulset_test.go | 121 +++ internal/utils/store.go | 110 ++ internal/utils/store_test.go | 118 +++ internal/utils/utils_suite_test.go | 27 + internal/version/version.go | 24 + 85 files changed, 11156 insertions(+), 3 deletions(-) create mode 100644 internal/client/kubernetes/types.go create mode 100644 internal/common/constants.go create mode 100644 internal/controller/compaction/config.go create mode 100644 internal/controller/compaction/metrics.go create mode 100644 internal/controller/compaction/reconciler.go create mode 100644 internal/controller/compaction/register.go create mode 100644 internal/controller/config.go create mode 100644 internal/controller/etcd/config.go create mode 100644 internal/controller/etcd/reconcile_delete.go create mode 100644 internal/controller/etcd/reconcile_spec.go create mode 100644 internal/controller/etcd/reconciler.go create mode 100644 internal/controller/etcd/register.go create mode 100644 internal/controller/etcdcopybackupstask/config.go create mode 100644 internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go create mode 100644 internal/controller/etcdcopybackupstask/reconciler.go create mode 100644 internal/controller/etcdcopybackupstask/reconciler_test.go create mode 100644 internal/controller/etcdcopybackupstask/register.go create mode 100644 internal/controller/manager.go create mode 100644 internal/controller/predicate/predicate.go create mode 100644 internal/controller/predicate/predicate_suite_test.go create mode 100644 internal/controller/predicate/predicate_test.go create mode 100644 internal/controller/secret/config.go create mode 100644 internal/controller/secret/reconciler.go create mode 100644 internal/controller/secret/reconciler_test.go create mode 100644 internal/controller/secret/register.go create mode 100644 internal/controller/secret/secret_suite_test.go create mode 100644 internal/controller/utils/etcdstatus.go create mode 100644 internal/controller/utils/reconciler.go create mode 100644 internal/controller/utils/utils_suite_test.go create mode 100644 internal/controller/utils/validator.go create mode 100644 internal/controller/utils/validator_test.go create mode 100644 internal/errors/errors.go create mode 100644 internal/features/features.go create mode 100644 internal/health/condition/builder.go create mode 100644 internal/health/condition/builder_test.go create mode 100644 internal/health/condition/check_all_members.go create mode 100644 internal/health/condition/check_all_members_test.go create mode 100644 internal/health/condition/check_backup_ready.go create mode 100644 internal/health/condition/check_backup_ready_test.go create mode 100644 internal/health/condition/check_ready.go create mode 100644 internal/health/condition/check_ready_test.go create mode 100644 internal/health/condition/condition_suite_test.go create mode 100644 internal/health/condition/types.go create mode 100644 internal/health/etcdmember/builder.go create mode 100644 internal/health/etcdmember/builder_test.go create mode 100644 internal/health/etcdmember/check_ready.go create mode 100644 internal/health/etcdmember/check_ready_test.go create mode 100644 internal/health/etcdmember/etcdmember_suite_test.go create mode 100644 internal/health/etcdmember/types.go create mode 100644 internal/health/status/check.go create mode 100644 internal/health/status/check_test.go create mode 100644 internal/health/status/status_suite_test.go create mode 100644 internal/mapper/etcd_to_secret.go create mode 100644 internal/mapper/etcd_to_secret_test.go create mode 100644 internal/mapper/mapper_suite_test.go create mode 100644 internal/mapper/statefulset_to_etcd.go create mode 100644 internal/mapper/statefulset_to_etcd_test.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/metrics/metrics_suite_test.go create mode 100644 internal/metrics/metrics_test.go create mode 100644 internal/mock/controller-runtime/client/doc.go create mode 100644 internal/mock/controller-runtime/client/mocks.go create mode 100644 internal/resource/clientservice/clientservice.go create mode 100644 internal/resource/configmap/configmap.go create mode 100644 internal/resource/configmap/etcdconfig.go create mode 100644 internal/resource/memberlease/memberlease.go create mode 100644 internal/resource/peerservice/peerservice.go create mode 100644 internal/resource/poddistruptionbudget/poddisruptionbudget.go create mode 100644 internal/resource/resource.go create mode 100644 internal/resource/role/role.go create mode 100644 internal/resource/rolebinding/rolebinding.go create mode 100644 internal/resource/serviceaccount/serviceaccount.go create mode 100644 internal/resource/snapshotlease/snapshotlease.go create mode 100644 internal/utils/concurrent.go create mode 100755 internal/utils/image.go create mode 100644 internal/utils/image_test.go create mode 100644 internal/utils/lease.go create mode 100644 internal/utils/miscellaneous.go create mode 100644 internal/utils/statefulset.go create mode 100644 internal/utils/statefulset_test.go create mode 100644 internal/utils/store.go create mode 100644 internal/utils/store_test.go create mode 100644 internal/utils/utils_suite_test.go create mode 100644 internal/version/version.go diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index e7f70e011..e3c8b99fb 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -10,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" ) @@ -33,7 +34,7 @@ const ( ZlibCompression CompressionPolicy = "zlib" // DefaultCompression is constant for default compression policy(only if compression is enabled). - DefaultCompression CompressionPolicy = GzipCompression + DefaultCompression = GzipCompression // DefaultCompressionEnabled is constant to define whether to compress the snapshots or not. DefaultCompressionEnabled = false @@ -381,8 +382,14 @@ type EtcdStatus struct { // +optional ServiceName *string `json:"serviceName,omitempty"` // LastError represents the last occurred error. + // Deprecated: Use LastErrors instead. // +optional LastError *string `json:"lastError,omitempty"` + // LastErrors captures errors that occurred during the last operation. + // +optional + LastErrors []LastError + // LastOperation indicates the last operation performed on this resource. + LastOperation *LastOperation // Cluster size is the current size of the etcd cluster. // Deprecated: this field will not be populated with any value and will be removed in the future. // +optional @@ -416,6 +423,68 @@ type EtcdStatus struct { PeerUrlTLSEnabled *bool `json:"peerUrlTLSEnabled,omitempty"` } +// LastOperationType is a string alias representing type of the last operation. +type LastOperationType string + +const ( + // LastOperationTypeCreate indicates that the last operation was a creation of a new etcd resource. + LastOperationTypeCreate LastOperationType = "Create" + // LastOperationTypeReconcile indicates that the last operation was a reconciliation of the spec of an etcd resource. + LastOperationTypeReconcile LastOperationType = "Reconcile" + // LastOperationTypeDelete indicates that the last operation was a deletion of an existing etcd resource. + LastOperationTypeDelete LastOperationType = "Delete" +) + +// LastOperationState is a string alias representing the state of the last operation. +type LastOperationState string + +const ( + // LastOperationStateProcessing indicates that an operation is in progress. + LastOperationStateProcessing LastOperationState = "Processing" + // LastOperationStateSucceeded indicates that an operation has completed successfully. + LastOperationStateSucceeded LastOperationState = "Succeeded" + // LastOperationStateError indicates that an operation is completed with errors and will be retried. + LastOperationStateError LastOperationState = "Error" +) + +// LastOperation holds the information on the last operation done on this resource. +type LastOperation struct { + // Type is the type of last operation. + Type LastOperationType `json:"type"` + // State is the state of the last operation. + State LastOperationState `json:"state"` + // Description describes the last operation. + Description string `json:"description"` + // RunID correlates an operation with a reconciliation run. + // Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is + // generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this + // as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering + // reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. + RunID string + // LastUpdateTime is the time at which the operation was updated. + LastUpdateTime metav1.Time `json:"lastUpdateTime"` +} + +// ErrorCode is a string alias representing an error code that identifies an error. +type ErrorCode string + +type LastError struct { + // Code is an error code that uniquely identifies an error. + Code ErrorCode + // Description is a human-readable message indicating details of the error. + Description string + // LastUpdateTime is the time the error was reported. + LastUpdateTime metav1.Time +} + +// GetNamespaceName is a convenience function which creates a types.NamespacedName for an etcd resource. +func (e *Etcd) GetNamespaceName() types.NamespacedName { + return types.NamespacedName{ + Namespace: e.Namespace, + Name: e.Name, + } +} + // GetPeerServiceName returns the peer service name for the Etcd cluster reachable by members within the Etcd cluster. func (e *Etcd) GetPeerServiceName() string { return fmt.Sprintf("%s-peer", e.Name) @@ -431,8 +500,8 @@ func (e *Etcd) GetServiceAccountName() string { return e.Name } -// GetConfigmapName returns the name of the configmap for the Etcd. -func (e *Etcd) GetConfigmapName() string { +// GetConfigMapName returns the name of the configmap for the Etcd. +func (e *Etcd) GetConfigMapName() string { return fmt.Sprintf("etcd-bootstrap-%s", string(e.UID[:6])) } @@ -456,6 +525,16 @@ func (e *Etcd) GetFullSnapshotLeaseName() string { return fmt.Sprintf("%s-full-snap", e.Name) } +// GetMemberLeaseNames returns the name of member leases for the Etcd. +func (e *Etcd) GetMemberLeaseNames() []string { + numReplicas := int(e.Spec.Replicas) + leaseNames := make([]string, 0, numReplicas) + for i := 0; i < numReplicas; i++ { + leaseNames = append(leaseNames, fmt.Sprintf("%s-%d", e.Name, i)) + } + return leaseNames +} + // GetDefaultLabels returns the default labels for etcd. func (e *Etcd) GetDefaultLabels() map[string]string { return map[string]string{ @@ -485,3 +564,13 @@ func (e *Etcd) GetRoleName() string { func (e *Etcd) GetRoleBindingName() string { return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, e.Name) } + +// IsBackupEnabled returns true if backup has been enabled for this etcd, else returns false. +func (e *Etcd) IsBackupEnabled() bool { + return e.Spec.Backup.Store != nil +} + +// IsMarkedForDeletion returns true if a deletion timestamp has been set and false otherwise. +func (e *Etcd) IsMarkedForDeletion() bool { + return !e.DeletionTimestamp.IsZero() +} diff --git a/internal/client/kubernetes/types.go b/internal/client/kubernetes/types.go new file mode 100644 index 000000000..3fe5155ec --- /dev/null +++ b/internal/client/kubernetes/types.go @@ -0,0 +1,35 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + schemev1 "k8s.io/client-go/kubernetes/scheme" +) + +// Scheme is the kubernetes runtime scheme +var Scheme = runtime.NewScheme() + +func init() { + localSchemeBuilder := runtime.NewSchemeBuilder( + druidv1alpha1.AddToScheme, + schemev1.AddToScheme, + ) + + utilruntime.Must(localSchemeBuilder.AddToScheme(Scheme)) +} diff --git a/internal/common/constants.go b/internal/common/constants.go new file mode 100644 index 000000000..0882c630f --- /dev/null +++ b/internal/common/constants.go @@ -0,0 +1,48 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +const ( + // Etcd is the key for the etcd image in the image vector. + Etcd = "etcd" + // BackupRestore is the key for the etcd-backup-restore image in the image vector. + BackupRestore = "etcd-backup-restore" + // EtcdWrapper is the key for the etcd image in the image vector. + EtcdWrapper = "etcd-wrapper" + // BackupRestoreDistroless is the key for the etcd-backup-restore image in the image vector. + BackupRestoreDistroless = "etcd-backup-restore-distroless" + // ChartPath is the directory containing the default image vector file. + ChartPath = "charts" + // GardenerOwnedBy is a constant for an annotation on a resource that describes the owner resource. + GardenerOwnedBy = "gardener.cloud/owned-by" + // GardenerOwnerType is a constant for an annotation on a resource that describes the type of owner resource. + GardenerOwnerType = "gardener.cloud/owner-type" + // FinalizerName is the name of the etcd finalizer. + FinalizerName = "druid.gardener.cloud/etcd-druid" + // STORAGE_CONTAINER is the environment variable key for the storage container. + STORAGE_CONTAINER = "STORAGE_CONTAINER" + // AWS_APPLICATION_CREDENTIALS is the environment variable key for AWS application credentials. + AWS_APPLICATION_CREDENTIALS = "AWS_APPLICATION_CREDENTIALS" + // AZURE_APPLICATION_CREDENTIALS is the environment variable key for Azure application credentials. + AZURE_APPLICATION_CREDENTIALS = "AZURE_APPLICATION_CREDENTIALS" + // GOOGLE_APPLICATION_CREDENTIALS is the environment variable key for Google application credentials. + GOOGLE_APPLICATION_CREDENTIALS = "GOOGLE_APPLICATION_CREDENTIALS" + // OPENSTACK_APPLICATION_CREDENTIALS is the environment variable key for OpenStack application credentials. + OPENSTACK_APPLICATION_CREDENTIALS = "OPENSTACK_APPLICATION_CREDENTIALS" + // OPENSHIFT_APPLICATION_CREDENTIALS is the environment variable key for OpenShift application credentials. + OPENSHIFT_APPLICATION_CREDENTIALS = "OPENSHIFT_APPLICATION_CREDENTIALS" + // ALICLOUD_APPLICATION_CREDENTIALS is the environment variable key for Alicloud application credentials. + ALICLOUD_APPLICATION_CREDENTIALS = "ALICLOUD_APPLICATION_CREDENTIALS" +) diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go new file mode 100644 index 000000000..0c7d3e634 --- /dev/null +++ b/internal/controller/compaction/config.go @@ -0,0 +1,95 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compaction + +import ( + "time" + + "github.com/gardener/etcd-druid/controllers/utils" + "github.com/gardener/etcd-druid/pkg/features" + + flag "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +// featureList holds the feature gate names that are relevant for the Compaction Controller. +var featureList = []featuregate.Feature{ + features.UseEtcdWrapper, +} + +const ( + enableBackupCompactionFlagName = "enable-backup-compaction" + workersFlagName = "compaction-workers" + eventsThresholdFlagName = "etcd-events-threshold" + activeDeadlineDurationFlagName = "active-deadline-duration" + metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" + + defaultEnableBackupCompaction = false + defaultCompactionWorkers = 3 + defaultEventsThreshold = 1000000 + defaultActiveDeadlineDuration = 3 * time.Hour + defaultMetricsScrapeWaitDuration = 0 +) + +// Config contains configuration for the Compaction Controller. +type Config struct { + // EnableBackupCompaction specifies whether backup compaction should be enabled. + EnableBackupCompaction bool + // Workers denotes the number of worker threads for the compaction controller. + Workers int + // EventsThreshold denotes total number of etcd events to be reached upon which a backup compaction job is triggered. + EventsThreshold int64 + // ActiveDeadlineDuration is the duration after which a running compaction job will be killed. + ActiveDeadlineDuration time.Duration + // MetricsScrapeWaitDuration is the duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped + MetricsScrapeWaitDuration time.Duration + // FeatureGates contains the feature gates to be used by Compaction Controller. + FeatureGates map[featuregate.Feature]bool +} + +// InitFromFlags initializes the compaction controller config from the provided CLI flag set. +func InitFromFlags(fs *flag.FlagSet, cfg *Config) { + fs.BoolVar(&cfg.EnableBackupCompaction, enableBackupCompactionFlagName, defaultEnableBackupCompaction, + "Enable automatic compaction of etcd backups.") + fs.IntVar(&cfg.Workers, workersFlagName, defaultCompactionWorkers, + "Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. Setting this flag to 0 disables the controller.") + fs.Int64Var(&cfg.EventsThreshold, eventsThresholdFlagName, defaultEventsThreshold, + "Total number of etcd events that can be allowed before a backup compaction job is triggered.") + fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, + "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\").") + fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagname, defaultMetricsScrapeWaitDuration, + "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\").") +} + +// Validate validates the config. +func (cfg *Config) Validate() error { + if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, 0, cfg.Workers); err != nil { + return err + } + if err := utils.MustBeGreaterThan(eventsThresholdFlagName, 0, cfg.EventsThreshold); err != nil { + return err + } + return utils.MustBeGreaterThan(activeDeadlineDurationFlagName, 0, cfg.ActiveDeadlineDuration.Seconds()) +} + +// CaptureFeatureActivations captures all feature gates required by the controller into controller config +func (cfg *Config) CaptureFeatureActivations(fg featuregate.FeatureGate) { + if cfg.FeatureGates == nil { + cfg.FeatureGates = make(map[featuregate.Feature]bool) + } + for _, feature := range featureList { + cfg.FeatureGates[feature] = fg.Enabled(feature) + } +} diff --git a/internal/controller/compaction/metrics.go b/internal/controller/compaction/metrics.go new file mode 100644 index 000000000..fa7354e08 --- /dev/null +++ b/internal/controller/compaction/metrics.go @@ -0,0 +1,112 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compaction + +import ( + "time" + + druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + namespaceEtcdDruid = "etcddruid" + subsystemCompaction = "compaction" +) + +var ( + baseDuration = 60 * time.Second + // metricJobsTotal is the metric used to count the total number of compaction jobs initiated by compaction controller. + metricJobsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespaceEtcdDruid, + Subsystem: subsystemCompaction, + Name: "jobs_total", + Help: "Total number of compaction jobs initiated by compaction controller.", + }, + []string{druidmetrics.LabelSucceeded, druidmetrics.EtcdNamespace}, + ) + + // metricJobsCurrent is the metric used to expose currently running compaction job. This metric is important to get a sense of total number of compaction job running in a seed cluster. + metricJobsCurrent = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespaceEtcdDruid, + Subsystem: subsystemCompaction, + Name: "jobs_current", + Help: "Number of currently running comapction job.", + }, + []string{druidmetrics.EtcdNamespace}, + ) + + // metricJobDurationSeconds is the metric used to expose the time taken to finish a compaction job. + metricJobDurationSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespaceEtcdDruid, + Subsystem: subsystemCompaction, + Name: "job_duration_seconds", + Help: "Total time taken in seconds to finish a running compaction job.", + Buckets: []float64{baseDuration.Seconds(), + 5 * baseDuration.Seconds(), + 10 * baseDuration.Seconds(), + 15 * baseDuration.Seconds(), + 20 * baseDuration.Seconds(), + 30 * baseDuration.Seconds(), + }, + }, + []string{druidmetrics.LabelSucceeded, druidmetrics.EtcdNamespace}, + ) + + // metricNumDeltaEvents is the metric used to expose the total number of etcd events to be compacted by a compaction job. + metricNumDeltaEvents = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespaceEtcdDruid, + Subsystem: subsystemCompaction, + Name: "num_delta_events", + Help: "Total number of etcd events to be compacted by a compaction job.", + }, + []string{druidmetrics.EtcdNamespace}, + ) +) + +func init() { + // metricJobsTotalValues + metricJobsTotalValues := map[string][]string{ + druidmetrics.LabelSucceeded: druidmetrics.DruidLabels[druidmetrics.LabelSucceeded], + druidmetrics.EtcdNamespace: druidmetrics.DruidLabels[druidmetrics.EtcdNamespace], + } + metricJobsTotalCombinations := druidmetrics.GenerateLabelCombinations(metricJobsTotalValues) + for _, combination := range metricJobsTotalCombinations { + metricJobsTotal.With(prometheus.Labels(combination)) + } + + // metricJobDurationSecondsValues + metricJobDurationSecondsValues := map[string][]string{ + druidmetrics.LabelSucceeded: druidmetrics.DruidLabels[druidmetrics.LabelSucceeded], + druidmetrics.EtcdNamespace: druidmetrics.DruidLabels[druidmetrics.EtcdNamespace], + } + compactionJobDurationSecondsCombinations := druidmetrics.GenerateLabelCombinations(metricJobDurationSecondsValues) + for _, combination := range compactionJobDurationSecondsCombinations { + metricJobDurationSeconds.With(prometheus.Labels(combination)) + } + + // Metrics have to be registered to be exposed: + metrics.Registry.MustRegister(metricJobsTotal) + metrics.Registry.MustRegister(metricJobDurationSeconds) + + metrics.Registry.MustRegister(metricJobsCurrent) + + metrics.Registry.MustRegister(metricNumDeltaEvents) +} diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go new file mode 100644 index 000000000..cab28f1d6 --- /dev/null +++ b/internal/controller/compaction/reconciler.go @@ -0,0 +1,558 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compaction + +import ( + "context" + "fmt" + "strconv" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + druidmetrics "github.com/gardener/etcd-druid/internal/metrics" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/etcd-druid/pkg/features" + "github.com/gardener/gardener/pkg/utils/imagevector" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + // DefaultETCDQuota is the default etcd quota. + DefaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi +) + +// Reconciler reconciles compaction jobs for Etcd resources. +type Reconciler struct { + client.Client + config *Config + imageVector imagevector.ImageVector + logger logr.Logger +} + +// NewReconciler creates a new reconciler for Compaction +func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { + imageVector, err := ctrlutils.CreateImageVector() + if err != nil { + return nil, err + } + return NewReconcilerWithImageVector(mgr, config, imageVector), nil +} + +// NewReconcilerWithImageVector creates a new reconciler for Compaction with an ImageVector. +// This constructor will mostly be used by tests. +func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { + return &Reconciler{ + Client: mgr.GetClient(), + config: config, + imageVector: imageVector, + logger: log.Log.WithName("compaction-lease-controller"), + } +} + +// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;create;list;watch;update;patch;delete +// +kubebuilder:rbac:groups="",resources=pods,verbs=list;watch;delete;get + +// Reconcile reconciles the compaction job. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.logger.Info("Compaction job reconciliation started") + etcd := &druidv1alpha1.Etcd{} + if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + if !etcd.DeletionTimestamp.IsZero() || etcd.Spec.Backup.Store == nil { + // Delete compaction job if exists + return r.delete(ctx, r.logger, etcd) + } + + logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) + + return r.reconcileJob(ctx, logger, etcd) +} + +func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { + // Update metrics for currently running compaction job, if any + job := &batchv1.Job{} + if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { + if errors.IsNotFound(err) { + logger.Info("Currently, no compaction job is running in the namespace ", etcd.Namespace) + } else { + // Error reading the object - requeue the request. + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", etcd.GetCompactionJobName(), etcd.Namespace, err) + } + } + + if job != nil && job.Name != "" { + if !job.DeletionTimestamp.IsZero() { + logger.Info("Job is already in deletion. A new job will be created only if the previous one has been deleted.", "namespace: ", job.Namespace, "name: ", job.Name) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil + } + + // Check if there is one active job or not + if job.Status.Active > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) + // Don't need to requeue if the job is currently running + return ctrl.Result{}, nil + } + + // Delete job if the job succeeded + if job.Status.Succeeded > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) + if job.Status.CompletionTime != nil { + metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) + } + if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { + logger.Error(err, "Couldn't delete the successful job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while deleting successful compaction job: %v", err) + } + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() + } + + // Delete job and requeue if the job failed + if job.Status.Failed > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) + if job.Status.StartTime != nil { + metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) + } + err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) + if err != nil { + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while deleting failed compaction job: %v", err) + } + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil + } + } + + // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job + fullLease := &coordinationv1.Lease{} + + if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetFullSnapshotLeaseName()), fullLease); err != nil { + logger.Error(err, "Couldn't fetch full snap lease", "namespace", etcd.Namespace, "name", etcd.GetFullSnapshotLeaseName()) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + deltaLease := &coordinationv1.Lease{} + if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetDeltaSnapshotLeaseName()), deltaLease); err != nil { + logger.Error(err, "Couldn't fetch delta snap lease", "namespace", etcd.Namespace, "name", etcd.GetDeltaSnapshotLeaseName()) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + // Revisions have not been set yet by etcd-back-restore container. + // Skip further processing as we cannot calculate a revision delta. + if fullLease.Spec.HolderIdentity == nil || deltaLease.Spec.HolderIdentity == nil { + return ctrl.Result{}, nil + } + + full, err := strconv.ParseInt(*fullLease.Spec.HolderIdentity, 10, 64) + if err != nil { + logger.Error(err, "Can't convert holder identity of full snap lease to integer", + "namespace", fullLease.Namespace, "leaseName", fullLease.Name, "holderIdentity", fullLease.Spec.HolderIdentity) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + delta, err := strconv.ParseInt(*deltaLease.Spec.HolderIdentity, 10, 64) + if err != nil { + logger.Error(err, "Can't convert holder identity of delta snap lease to integer", + "namespace", deltaLease.Namespace, "leaseName", deltaLease.Name, "holderIdentity", deltaLease.Spec.HolderIdentity) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + diff := delta - full + metricNumDeltaEvents.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(float64(diff)) + + // Reconcile job only when number of accumulated revisions over the last full snapshot is more than the configured threshold value via 'events-threshold' flag + if diff >= r.config.EventsThreshold { + logger.Info("Creating etcd compaction job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) + job, err = r.createCompactionJob(ctx, logger, etcd) + if err != nil { + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error during compaction job creation: %v", err) + } + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) + } + + if job.Name != "" { + logger.Info("Current compaction job status", + "namespace", job.Namespace, "name", job.Name, "succeeded", job.Status.Succeeded) + } + + return ctrl.Result{Requeue: false}, nil +} + +func (r *Reconciler) delete(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction job: %v", err) + } + return ctrl.Result{Requeue: false}, nil + } + + if job.DeletionTimestamp == nil { + logger.Info("Deleting job", "namespace", job.Namespace, "name", job.Name) + if err := client.IgnoreNotFound(r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while deleting compaction job: %v", err) + } + } + + logger.Info("No compaction job is running") + return ctrl.Result{ + Requeue: false, + }, nil +} + +func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*batchv1.Job, error) { + activeDeadlineSeconds := r.config.ActiveDeadlineDuration.Seconds() + + _, etcdBackupImage, _, err := utils.GetEtcdImages(etcd, r.imageVector, r.config.FeatureGates[features.UseEtcdWrapper]) + if err != nil { + return nil, fmt.Errorf("couldn't fetch etcd backup image: %v", err) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetCompactionJobName(), + Namespace: etcd.Namespace, + Labels: getLabels(etcd), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: druidv1alpha1.GroupVersion.String(), + BlockOwnerDeletion: pointer.Bool(true), + Controller: pointer.Bool(true), + Kind: "Etcd", + Name: etcd.Name, + UID: etcd.UID, + }, + }, + }, + + Spec: batchv1.JobSpec{ + ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), + Completions: pointer.Int32(1), + BackoffLimit: pointer.Int32(0), + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: etcd.Spec.Annotations, + Labels: getLabels(etcd), + }, + Spec: v1.PodSpec{ + ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), + ServiceAccountName: etcd.GetServiceAccountName(), + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{{ + Name: "compact-backup", + Image: *etcdBackupImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: getCompactionJobArgs(etcd, r.config.MetricsScrapeWaitDuration.String()), + }}, + }, + }, + }, + } + + if vms, err := getCompactionJobVolumeMounts(etcd); err != nil { + return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", + etcd.Namespace, + etcd.Name, + err) + } else { + job.Spec.Template.Spec.Containers[0].VolumeMounts = vms + } + + if env, err := getCompactionJobEnvVar(etcd); err != nil { + return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", + etcd.Namespace, + etcd.Name, + err) + } else { + job.Spec.Template.Spec.Containers[0].Env = env + } + + if vm, err := getCompactionJobVolumes(ctx, r.Client, r.logger, etcd); err != nil { + return nil, fmt.Errorf("error creating compaction job in %v for %v : %v", + etcd.Namespace, + etcd.Name, + err) + } else { + job.Spec.Template.Spec.Volumes = vm + } + + if etcd.Spec.Backup.CompactionResources != nil { + job.Spec.Template.Spec.Containers[0].Resources = *etcd.Spec.Backup.CompactionResources + } + + logger.Info("Creating job", "namespace", job.Namespace, "name", job.Name) + err = r.Create(ctx, job) + if err != nil { + return nil, err + } + + //TODO (abdasgupta): Evaluate necessity of claiming object here after creation + return job, nil +} + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + return map[string]string{ + "name": "etcd-backup-compaction", + "instance": etcd.Name, + "gardener.cloud/role": "controlplane", + "networking.gardener.cloud/to-dns": "allowed", + "networking.gardener.cloud/to-private-networks": "allowed", + "networking.gardener.cloud/to-public-networks": "allowed", + } +} +func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd) ([]v1.VolumeMount, error) { + vms := []v1.VolumeMount{ + { + Name: "etcd-workspace-dir", + MountPath: "/var/etcd/data", + }, + } + + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return vms, fmt.Errorf("storage provider is not recognized while fetching volume mounts") + } + switch provider { + case utils.Local: + vms = append(vms, v1.VolumeMount{ + Name: "host-storage", + MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), + }) + case utils.GCS: + vms = append(vms, v1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/.gcp/", + }) + case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + vms = append(vms, v1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/etcd-backup/", + }) + } + + return vms, nil +} + +func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr.Logger, etcd *druidv1alpha1.Etcd) ([]v1.Volume, error) { + vs := []v1.Volume{ + { + Name: "etcd-workspace-dir", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + + storeValues := etcd.Spec.Backup.Store + provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + if err != nil { + return vs, fmt.Errorf("could not recognize storage provider while fetching volumes") + } + switch provider { + case "Local": + hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, etcd.Namespace) + if err != nil { + return vs, fmt.Errorf("could not determine host mount path for local provider") + } + + hpt := v1.HostPathDirectory + vs = append(vs, v1.Volume{ + Name: "host-storage", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), + Type: &hpt, + }, + }, + }) + case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + if storeValues.SecretRef == nil { + return vs, fmt.Errorf("could not configure secretRef for backup store %v", provider) + } + + vs = append(vs, v1.Volume{ + Name: "etcd-backup", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: storeValues.SecretRef.Name, + }, + }, + }) + } + + return vs, nil +} + +func getCompactionJobEnvVar(etcd *druidv1alpha1.Etcd) ([]v1.EnvVar, error) { + var env []v1.EnvVar + + storeValues := etcd.Spec.Backup.Store + + env = append(env, getEnvVarFromValues("STORAGE_CONTAINER", *storeValues.Container)) + env = append(env, getEnvVarFromFields("POD_NAMESPACE", "metadata.namespace")) + + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return env, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") + } + + switch provider { + case utils.S3: + env = append(env, getEnvVarFromValues("AWS_APPLICATION_CREDENTIALS", "/var/etcd-backup")) + case utils.ABS: + env = append(env, getEnvVarFromValues("AZURE_APPLICATION_CREDENTIALS", "/var/etcd-backup")) + case utils.GCS: + env = append(env, getEnvVarFromValues("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) + case utils.Swift: + env = append(env, getEnvVarFromValues("OPENSTACK_APPLICATION_CREDENTIALS", "/var/etcd-backup")) + case utils.OSS: + env = append(env, getEnvVarFromValues("ALICLOUD_APPLICATION_CREDENTIALS", "/var/etcd-backup")) + case utils.ECS: + if storeValues.SecretRef == nil { + return env, fmt.Errorf("no secretRef could be configured for backup store of ECS") + } + + env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) + env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) + env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) + case utils.OCS: + env = append(env, getEnvVarFromValues("OPENSHIFT_APPLICATION_CREDENTIALS", "/var/etcd-backup")) + } + + return env, nil +} + +func getEnvVarFromValues(name, value string) v1.EnvVar { + return v1.EnvVar{ + Name: name, + Value: value, + } +} + +func getEnvVarFromFields(name, fieldPath string) v1.EnvVar { + return v1.EnvVar{ + Name: name, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: fieldPath, + }, + }, + } +} + +func getEnvVarFromSecrets(name, secretName, secretKey string) v1.EnvVar { + return v1.EnvVar{ + Name: name, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: secretName, + }, + Key: secretKey, + }, + }, + } +} + +func getCompactionJobArgs(etcd *druidv1alpha1.Etcd, metricsScrapeWaitDuration string) []string { + command := []string{"compact"} + command = append(command, "--data-dir=/var/etcd/data/compaction.etcd") + command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp") + command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") + command = append(command, "--metrics-scrape-wait-duration="+metricsScrapeWaitDuration) + command = append(command, "--enable-snapshot-lease-renewal=true") + command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) + command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) + + var quota int64 = DefaultETCDQuota + if etcd.Spec.Etcd.Quota != nil { + quota = etcd.Spec.Etcd.Quota.Value() + } + command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) + + if etcd.Spec.Etcd.EtcdDefragTimeout != nil { + command = append(command, "--etcd-defrag-timeout="+etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String()) + } + + backupValues := etcd.Spec.Backup + if backupValues.EtcdSnapshotTimeout != nil { + command = append(command, "--etcd-snapshot-timeout="+backupValues.EtcdSnapshotTimeout.Duration.String()) + } + storeValues := etcd.Spec.Backup.Store + if storeValues != nil { + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err == nil { + command = append(command, "--storage-provider="+provider) + } + + if storeValues.Prefix != "" { + command = append(command, "--store-prefix="+storeValues.Prefix) + } + + if storeValues.Container != nil { + command = append(command, "--store-container="+*(storeValues.Container)) + } + } + + return command +} diff --git a/internal/controller/compaction/register.go b/internal/controller/compaction/register.go new file mode 100644 index 000000000..e9921cb32 --- /dev/null +++ b/internal/controller/compaction/register.go @@ -0,0 +1,45 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compaction + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druidpredicates "github.com/gardener/etcd-druid/internal/controller/predicate" + + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +const controllerName = "compaction-controller" + +// RegisterWithManager registers the Compaction Controller with the given controller manager. +func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { + return ctrl. + NewControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.config.Workers, + }). + For(&druidv1alpha1.Etcd{}). + WithEventFilter(predicate. + Or(druidpredicates.SnapshotRevisionChanged(), + druidpredicates.JobStatusChanged())). + Owns(&coordinationv1.Lease{}). + Owns(&batchv1.Job{}). + Complete(r) +} diff --git a/internal/controller/config.go b/internal/controller/config.go new file mode 100644 index 000000000..b53dec2c6 --- /dev/null +++ b/internal/controller/config.go @@ -0,0 +1,172 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "fmt" + + "github.com/gardener/etcd-druid/controllers/custodian" + "github.com/gardener/etcd-druid/internal/controller/compaction" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" + "github.com/gardener/etcd-druid/internal/controller/secret" + "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/features" + flag "github.com/spf13/pflag" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/component-base/featuregate" +) + +const ( + metricsAddrFlagName = "metrics-addr" + enableLeaderElectionFlagName = "enable-leader-election" + leaderElectionIDFlagName = "leader-election-id" + leaderElectionResourceLockFlagName = "leader-election-resource-lock" + disableLeaseCacheFlagName = "disable-lease-cache" + + defaultMetricsAddr = ":8080" + defaultEnableLeaderElection = false + defaultLeaderElectionID = "druid-leader-election" + defaultLeaderElectionResourceLock = resourcelock.LeasesResourceLock + defaultDisableLeaseCache = false +) + +// LeaderElectionConfig defines the configuration for the leader election for the controller manager. +type LeaderElectionConfig struct { + // EnableLeaderElection specifies whether to enable leader election for controller manager. + EnableLeaderElection bool + // LeaderElectionID is the name of the resource that leader election will use for holding the leader lock. + LeaderElectionID string + // LeaderElectionResourceLock specifies which resource type to use for leader election. + // Deprecated: K8S Leases will be used for leader election. No other resource type would be permitted. + // This configuration option will be removed eventually. It is advisable to not use this option any longer. + LeaderElectionResourceLock string +} + +// ManagerConfig defines the configuration for the controller manager. +type ManagerConfig struct { + // MetricsAddr is the address the metric endpoint binds to. + MetricsAddr string + LeaderElectionConfig + // DisableLeaseCache specifies whether to disable cache for lease.coordination.k8s.io resources. + DisableLeaseCache bool + // FeatureGates contains the feature gates to be used by etcd-druid. + FeatureGates featuregate.MutableFeatureGate + // EtcdControllerConfig is the configuration required for etcd controller. + EtcdControllerConfig *etcd.Config + // CustodianControllerConfig is the configuration required for custodian controller. + CustodianControllerConfig *custodian.Config + // CompactionControllerConfig is the configuration required for compaction controller. + CompactionControllerConfig *compaction.Config + // EtcdCopyBackupsTaskControllerConfig is the configuration required for etcd-copy-backup-tasks controller. + EtcdCopyBackupsTaskControllerConfig *etcdcopybackupstask.Config + // SecretControllerConfig is the configuration required for secret controller. + SecretControllerConfig *secret.Config +} + +// InitFromFlags initializes the controller manager config from the provided CLI flag set. +func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { + flag.StringVar(&cfg.MetricsAddr, metricsAddrFlagName, defaultMetricsAddr, ""+ + "The address the metric endpoint binds to.") + flag.BoolVar(&cfg.EnableLeaderElection, enableLeaderElectionFlagName, defaultEnableLeaderElection, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&cfg.LeaderElectionID, leaderElectionIDFlagName, defaultLeaderElectionID, + "Name of the resource that leader election will use for holding the leader lock") + flag.StringVar(&cfg.LeaderElectionResourceLock, leaderElectionResourceLockFlagName, defaultLeaderElectionResourceLock, + "Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.") + flag.BoolVar(&cfg.DisableLeaseCache, disableLeaseCacheFlagName, defaultDisableLeaseCache, + "Disable cache for lease.coordination.k8s.io resources.") + + if err := cfg.initFeatureGates(fs); err != nil { + return err + } + + cfg.EtcdControllerConfig = &etcd.Config{} + etcd.InitFromFlags(fs, cfg.EtcdControllerConfig) + + cfg.CustodianControllerConfig = &custodian.Config{} + custodian.InitFromFlags(fs, cfg.CustodianControllerConfig) + + cfg.CompactionControllerConfig = &compaction.Config{} + compaction.InitFromFlags(fs, cfg.CompactionControllerConfig) + + cfg.EtcdCopyBackupsTaskControllerConfig = &etcdcopybackupstask.Config{} + etcdcopybackupstask.InitFromFlags(fs, cfg.EtcdCopyBackupsTaskControllerConfig) + + cfg.SecretControllerConfig = &secret.Config{} + secret.InitFromFlags(fs, cfg.SecretControllerConfig) + + return nil +} + +// initFeatureGates initializes feature gates from the provided CLI flag set. +func (cfg *ManagerConfig) initFeatureGates(fs *flag.FlagSet) error { + featureGates := featuregate.NewFeatureGate() + if err := featureGates.Add(features.GetDefaultFeatures()); err != nil { + return fmt.Errorf("error adding features to the featuregate: %v", err) + } + featureGates.AddFlag(fs) + + cfg.FeatureGates = featureGates + + return nil +} + +// populateControllersFeatureGates adds relevant feature gates to every controller configuration +func (cfg *ManagerConfig) populateControllersFeatureGates() { + // Feature gates populated only for controllers that use feature gates + + // Add etcd controller feature gates + cfg.EtcdControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) + + // Add compaction controller feature gates + cfg.CompactionControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) + + // Add etcd-copy-backups-task controller feature gates + cfg.EtcdCopyBackupsTaskControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) +} + +// Validate validates the controller manager config. +func (cfg *ManagerConfig) Validate() error { + if err := utils.ShouldBeOneOfAllowedValues("LeaderElectionResourceLock", getAllowedLeaderElectionResourceLocks(), cfg.LeaderElectionResourceLock); err != nil { + return err + } + if err := cfg.EtcdControllerConfig.Validate(); err != nil { + return err + } + + if err := cfg.CustodianControllerConfig.Validate(); err != nil { + return err + } + + if err := cfg.CompactionControllerConfig.Validate(); err != nil { + return err + } + + if err := cfg.EtcdCopyBackupsTaskControllerConfig.Validate(); err != nil { + return err + } + + return cfg.SecretControllerConfig.Validate() +} + +// getAllowedLeaderElectionResourceLocks returns the allowed resource type to be used for leader election. +// TODO: This function should be removed as we have now marked 'leader-election-resource-lock' as deprecated. +// TODO: We will keep the validations till we have the CLI argument. Once that is removed we can also remove this function. +func getAllowedLeaderElectionResourceLocks() []string { + return []string{ + "leases", + } +} diff --git a/internal/controller/etcd/config.go b/internal/controller/etcd/config.go new file mode 100644 index 000000000..fb7b9fdbf --- /dev/null +++ b/internal/controller/etcd/config.go @@ -0,0 +1,92 @@ +package etcd + +import ( + "errors" + "time" + + "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/features" + flag "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +// Flag names +const ( + workersFlagName = "etcd-workers" + ignoreOperationAnnotationFlagName = "ignore-operation-annotation" + enableEtcdSpecAutoReconcileFlagName = "enable-etcd-spec-auto-reconcile" + disableEtcdServiceAccountAutomountFlagName = "disable-etcd-serviceaccount-automount" + etcdStatusSyncPeriodFlagName = "etcd-status-sync-period" +) + +const ( + defaultWorkers = 3 + defaultIgnoreOperationAnnotation = false + defaultDisableEtcdServiceAccountAutomount = false + defaultEnableEtcdSpecAutoReconcile = false + defaultEtcdStatusSyncPeriod = 15 * time.Second +) + +// featureList holds the feature gate names that are relevant for the Etcd Controller. +var featureList = []featuregate.Feature{ + features.UseEtcdWrapper, +} + +// Config defines the configuration for the Etcd Controller. +type Config struct { + // Workers is the number of workers concurrently processing reconciliation requests. + Workers int + // IgnoreOperationAnnotation specifies whether to ignore or honour the operation annotation on resources to be reconciled. + // Deprecated: Use EnableEtcdSpecAutoReconcile instead. + IgnoreOperationAnnotation bool + // EnableEtcdSpecAutoReconcile controls how the Etcd Spec is reconciled. If set to true, then any change in Etcd spec + // will automatically trigger a reconciliation of the Etcd resource. If set to false, then an operator needs to + // explicitly set gardener.cloud/operation=reconcile annotation on the Etcd resource to trigger reconciliation + // of the Etcd spec. + // NOTE: Decision to enable it should be carefully taken as spec updates could potentially result in rolling update + // of the StatefulSet which will cause a minor downtime for a single node etcd cluster and can potentially cause a + // downtime for a multi-node etcd cluster. + EnableEtcdSpecAutoReconcile bool + // DisableEtcdServiceAccountAutomount controls the auto-mounting of service account token for ETCD StatefulSets. + DisableEtcdServiceAccountAutomount bool + // EtcdStatusSyncPeriod is the duration after which an event will be re-queued ensuring ETCD status synchronization. + EtcdStatusSyncPeriod time.Duration + // FeatureGates contains the feature gates to be used by Etcd Controller. + FeatureGates map[featuregate.Feature]bool +} + +// InitFromFlags initializes the config from the provided CLI flag set. +func InitFromFlags(fs *flag.FlagSet, cfg *Config) { + fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, + "Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed.") + flag.BoolVar(&cfg.IgnoreOperationAnnotation, ignoreOperationAnnotationFlagName, defaultIgnoreOperationAnnotation, + "Specifies whether to ignore or honour the operation annotation on resources to be reconciled.") + flag.BoolVar(&cfg.EnableEtcdSpecAutoReconcile, enableEtcdSpecAutoReconcileFlagName, defaultEnableEtcdSpecAutoReconcile, + "If true then automatically reconciles Etcd Spec. If false waits for explicit annotation to be placed on the Etcd resource to trigger reconcile.") + fs.BoolVar(&cfg.DisableEtcdServiceAccountAutomount, disableEtcdServiceAccountAutomountFlagName, defaultDisableEtcdServiceAccountAutomount, + "If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets.") + fs.DurationVar(&cfg.EtcdStatusSyncPeriod, etcdStatusSyncPeriodFlagName, defaultEtcdStatusSyncPeriod, + "Period after which an etcd status sync will be attempted.") +} + +// Validate validates the config. +func (cfg *Config) Validate() error { + var errs []error + if err := utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers); err != nil { + errs = append(errs, err) + } + if err := utils.MustBeGreaterThan(etcdStatusSyncPeriodFlagName, 0, cfg.EtcdStatusSyncPeriod); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} + +// CaptureFeatureActivations captures all feature gates required by the controller into controller config +func (cfg *Config) CaptureFeatureActivations(fg featuregate.FeatureGate) { + if cfg.FeatureGates == nil { + cfg.FeatureGates = make(map[featuregate.Feature]bool) + } + for _, feature := range featureList { + cfg.FeatureGates[feature] = fg.Enabled(feature) + } +} diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go new file mode 100644 index 000000000..3cd1c7005 --- /dev/null +++ b/internal/controller/etcd/reconcile_delete.go @@ -0,0 +1,115 @@ +package etcd + +import ( + "fmt" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// deleteStepFn is a step in the deletion flow. Every deletion step must have this signature. +type deleteStepFn func(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult + +// triggerDeletionFlow is the entry point for the deletion flow triggered for an etcd resource which has a DeletionTimeStamp set on it. +func (r *Reconciler) triggerDeletionFlow(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { + deleteStepFns := []deleteStepFn{ + r.recordDeletionStartOperation, + r.deleteEtcdResources, + r.verifyNoResourcesAwaitCleanUp, + r.removeFinalizer, + } + for _, fn := range deleteStepFns { + if stepResult := fn(ctx, logger, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { + return r.recordIncompleteDeletionOperation(ctx, logger, etcdObjectKey, stepResult) + } + } + return ctrlutils.DoNotRequeue() +} + +func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + operators := r.operatorRegistry.AllOperators() + deleteTasks := make([]utils.OperatorTask, len(operators)) + for kind, operator := range operators { + operator := operator + deleteTasks = append(deleteTasks, utils.OperatorTask{ + Name: fmt.Sprintf("triggerDeletionFlow-%s-operator", kind), + Fn: func(ctx resource.OperatorContext) error { + return operator.TriggerDelete(ctx, etcd) + }, + }) + } + logger.Info("triggering triggerDeletionFlow operators for all resources") + if errs := utils.RunConcurrently(ctx, deleteTasks); len(errs) > 0 { + return ctrlutils.ReconcileWithError(errs...) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + operators := r.operatorRegistry.AllOperators() + resourceNamesAwaitingCleanup := make([]string, 0, len(operators)) + for _, operator := range operators { + existingResourceNames, err := operator.GetExistingResourceNames(ctx, etcd) + if err != nil { + return ctrlutils.ReconcileWithError(err) + } + resourceNamesAwaitingCleanup = append(resourceNamesAwaitingCleanup, existingResourceNames...) + } + if len(resourceNamesAwaitingCleanup) > 0 { + logger.Info("Cleanup of all resources has not yet been completed", "resourceNamesAwaitingCleanup", resourceNamesAwaitingCleanup) + return ctrlutils.ReconcileAfter(5*time.Second, "Cleanup of all resources has not yet been completed. Skipping removal of Finalizer") + } + logger.Info("All resources have been cleaned up") + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) removeFinalizer(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + logger.Info("Removing finalizer", "finalizerName", common.FinalizerName) + if err := controllerutils.RemoveFinalizers(ctx, r.client, etcd, common.FinalizerName); client.IgnoreNotFound(err) != nil { + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) recordDeletionStartOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if err := r.lastOpErrRecorder.RecordStart(ctx, etcd, druidv1alpha1.LastOperationTypeDelete); err != nil { + logger.Error(err, "failed to record etcd deletion start operation") + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) recordIncompleteDeletionOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if err := r.lastOpErrRecorder.RecordError(ctx, etcd, druidv1alpha1.LastOperationTypeDelete, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { + logger.Error(err, "failed to record last operation and last errors for etcd deletion") + return ctrlutils.ReconcileWithError(err) + } + return exitReconcileStepResult +} diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go new file mode 100644 index 000000000..bc4910602 --- /dev/null +++ b/internal/controller/etcd/reconcile_spec.go @@ -0,0 +1,71 @@ +package etcd + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +func (r *Reconciler) triggerReconcileSpec() error { + return nil +} + +func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { + //Check if spec reconciliation has been suspended, if yes, then record the event and return false. + if suspendReconcileAnnotKey := r.getSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil { + r.recordEtcdSpecReconcileSuspension(etcd, *suspendReconcileAnnotKey) + return false + } + // Check if + return true +} + +// getSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent +// to suspend spec reconciliation for this etcd resource. If no annotation is set then it will return nil. +func (r *Reconciler) getSuspendEtcdSpecReconcileAnnotationKey(etcd *druidv1alpha1.Etcd) *string { + var annotationKey *string + if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) { + annotationKey = pointer.String(druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) + } else if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.IgnoreReconciliationAnnotation) { + annotationKey = pointer.String(druidv1alpha1.IgnoreReconciliationAnnotation) + } + return annotationKey +} + +func (r *Reconciler) recordEtcdSpecReconcileSuspension(etcd *druidv1alpha1.Etcd, annotationKey string) { + r.recorder.Eventf( + etcd, + corev1.EventTypeWarning, + "SpecReconciliationSkipped", + "spec reconciliation of %s/%s is skipped by etcd-druid due to the presence of annotation %s on the etcd resource", + etcd.Namespace, + etcd.Name, + annotationKey, + ) +} + +//func (r *Reconciler) reconcileSpec(ctx context.Context, etcd *druidv1alpha1.Etcd, logger logr.Logger) utils.ReconcileStepResult { +// operatorCtx := resource.NewOperatorContext(ctx, r.client, logger, etcd.GetNamespaceName()) +// resourceOperators := r.getOrderedOperatorsForSync() +// for _, operator := range resourceOperators { +// if err := operator.Sync(operatorCtx, etcd); err != nil { +// return utils.ReconcileWithError(err) +// } +// } +// return utils.ContinueReconcile() +//} +// +//func (r *Reconciler) getOrderedOperatorsForSync() []resource.Operator { +// return []resource.Operator{ +// r.operatorRegistry.MemberLeaseOperator(), +// r.operatorRegistry.SnapshotLeaseOperator(), +// r.operatorRegistry.ClientServiceOperator(), +// r.operatorRegistry.PeerServiceOperator(), +// r.operatorRegistry.ConfigMapOperator(), +// r.operatorRegistry.PodDisruptionBudgetOperator(), +// r.operatorRegistry.ServiceAccountOperator(), +// r.operatorRegistry.RoleOperator(), +// r.operatorRegistry.RoleBindingOperator(), +// } +//} diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go new file mode 100644 index 000000000..934ab7fe2 --- /dev/null +++ b/internal/controller/etcd/reconciler.go @@ -0,0 +1,183 @@ +package etcd + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + "github.com/google/uuid" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +type Reconciler struct { + client client.Client + config *Config + recorder record.EventRecorder + imageVector imagevector.ImageVector + operatorRegistry resource.OperatorRegistry + lastOpErrRecorder ctrlutils.LastOperationErrorRecorder + logger logr.Logger +} + +// NewReconciler creates a new reconciler for Etcd. +func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { + imageVector, err := ctrlutils.CreateImageVector() + logger := log.Log.WithName(controllerName) + if err != nil { + return nil, err + } + operatorReg := resource.NewOperatorRegistry(mgr.GetClient(), + logger, + resource.OperatorConfig{ + DisableEtcdServiceAccountAutomount: config.DisableEtcdServiceAccountAutomount, + }, + ) + lastOpErrRecorder := ctrlutils.NewLastOperationErrorRecorder(mgr.GetClient(), logger) + return &Reconciler{ + client: mgr.GetClient(), + config: config, + recorder: mgr.GetEventRecorderFor(controllerName), + imageVector: imageVector, + logger: logger, + operatorRegistry: operatorReg, + lastOpErrRecorder: lastOpErrRecorder, + }, nil +} + +/* + If deletionTimestamp set: + triggerDeletionFlow(); if err then requeue + Else: + If ignore-reconciliation is set to true: + skip reconcileSpec() + Else If IgnoreOperationAnnotation flag is true: + always reconcileSpec() + Else If IgnoreOperationAnnotation flag is false and reconcile-op annotation is present: + reconcileSpec() + if err in getting etcd, return with requeue + if err in deploying any of the components, then record pending requeue + reconcileStatus() + requeue after minimum of X seconds (EtcdStatusSyncPeriod) and previously recorded requeue request +*/ + +// TODO: where/how is this being used? +// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds/status,verbs=get;create;update;patch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;watch +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;get;list + +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + type reconcileFn func(ctx context.Context, objectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult + reconcileFns := []reconcileFn{ + r.reconcileEtcdDeletion, + r.reconcileSpec, + r.reconcileStatus, + } + runID := uuid.New().String() + for _, fn := range reconcileFns { + if result := fn(ctx, req.NamespacedName, runID); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() + } + } + return ctrlutils.DoNotRequeue().ReconcileResult() +} + +func (r *Reconciler) reconcileEtcdDeletion(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if etcd.IsMarkedForDeletion() { + operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) + dLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "delete") + return r.triggerDeletionFlow(operatorCtx, dLog, etcdObjectKey) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) reconcileSpec(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + + return ctrlutils.DoNotRequeue() +} + +func (r *Reconciler) reconcileStatus(ctx context.Context, etcdNamespacedName types.NamespacedName, runID string) ctrlutils.ReconcileStepResult { + /* + fetch EtcdMember resources + fetch member leases + status.condition checks + status.members checks + fetch latest Etcd resource + update etcd status + */ + + return ctrlutils.DoNotRequeue() +} + +func (r *Reconciler) getLatestEtcd(ctx context.Context, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ctrlutils.ReconcileStepResult { + if err := r.client.Get(ctx, objectKey, etcd); err != nil { + if apierrors.IsNotFound(err) { + return ctrlutils.DoNotRequeue() + } + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() +} + +//func (r *Reconciler) runPreSpecReconcileChecks(etcd *druidv1alpha1.Etcd) ctrlutils.ReconcileStepResult { +// suspendEtcdSpecReconcileAnnotationKey := getSuspendEtcdReconcileAnnotationKey(etcd) +// if suspendEtcdSpecReconcileAnnotationKey != nil { +// r.recorder.Eventf( +// etcd, +// corev1.EventTypeWarning, +// "SpecReconciliationSkipped", +// "spec reconciliation of %s/%s is skipped by etcd-druid due to the presence of annotation %s on the etcd resource", +// etcd.Namespace, +// etcd.Name, +// suspendEtcdSpecReconcileAnnotationKey, +// ) +// +// } +//} + +//func (r *Reconciler) checkAndHandleReconcileSuspension(etcd *druidv1alpha1.Etcd) bool { +// //TODO: Once no one uses IgnoreReconciliationAnnotation annotation, then we can simplify this code. +// var annotationKey string +// if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) { +// annotationKey = druidv1alpha1.SuspendEtcdSpecReconcileAnnotation +// } else if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.IgnoreReconciliationAnnotation) { +// annotationKey = druidv1alpha1.IgnoreReconciliationAnnotation +// } +// if len(annotationKey) > 0 { +// r.recorder.Eventf( +// etcd, +// corev1.EventTypeWarning, +// "SpecReconciliationSkipped", +// "spec reconciliation of %s/%s is skipped by etcd-druid due to the presence of annotation %s on the etcd resource", +// etcd.Namespace, +// etcd.Name, +// annotationKey, +// ) +// return true +// } +// return ctrlutils.ContinueReconcile() +//} diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go new file mode 100644 index 000000000..27fae2f24 --- /dev/null +++ b/internal/controller/etcd/register.go @@ -0,0 +1,31 @@ +package etcd + +import ( + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +const controllerName = "etcd-controller" + +// RegisterWithManager registers the Etcd Controller with the given controller manager. +func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { + builder := ctrl. + NewControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.config.Workers, + // TODO: check if necessary + RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(10*time.Millisecond, 2*r.config.EtcdStatusSyncPeriod), + }). + For(&druidv1alpha1.Etcd{}) + + return builder.Complete(r) +} + +// TODO: create new etcd-recovery-controller which Owns (watches) all created resources +// If any of the owned resources is deleted/updated, and ignore-reconciliation annotation is not present on the etcd resource, +// then add the gardener.cloud/operation=reconcile on the etcd (if IgnoreOperationAnnotation is set to false) diff --git a/internal/controller/etcdcopybackupstask/config.go b/internal/controller/etcdcopybackupstask/config.go new file mode 100644 index 000000000..0290b7417 --- /dev/null +++ b/internal/controller/etcdcopybackupstask/config.go @@ -0,0 +1,63 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdcopybackupstask + +import ( + "github.com/gardener/etcd-druid/controllers/utils" + "github.com/gardener/etcd-druid/pkg/features" + + flag "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +// featureList holds the feature gate names that are relevant for the EtcdCopyBackupTask Controller. +var featureList = []featuregate.Feature{ + features.UseEtcdWrapper, +} + +const ( + workersFlagName = "etcd-copy-backups-task-workers" + + defaultWorkers = 3 +) + +// Config defines the configuration for the EtcdCopyBackupsTask Controller. +type Config struct { + // Workers is the number of workers concurrently processing reconciliation requests. + Workers int + // FeatureGates contains the feature gates to be used by EtcdCopyBackupTask Controller. + FeatureGates map[featuregate.Feature]bool +} + +// InitFromFlags initializes the config from the provided CLI flag set. +func InitFromFlags(fs *flag.FlagSet, cfg *Config) { + fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, + "Number of worker threads for the etcdcopybackupstask controller.") +} + +// Validate validates the config. +func (cfg *Config) Validate() error { + return utils.MustBeGreaterThanOrEqualTo(workersFlagName, 0, cfg.Workers) +} + +// CaptureFeatureActivations captures all feature gates required by the controller into controller config +func (cfg *Config) CaptureFeatureActivations(fg featuregate.FeatureGate) { + if cfg.FeatureGates == nil { + cfg.FeatureGates = make(map[featuregate.Feature]bool) + } + for _, feature := range featureList { + cfg.FeatureGates[feature] = fg.Enabled(feature) + } +} diff --git a/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go b/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go new file mode 100644 index 000000000..5ebe73b1a --- /dev/null +++ b/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdcopybackupstask + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestEtcdCopyBackupsTaskController(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs( + t, + "EtcdCopyBackupsTask Controller Suite", + ) +} diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go new file mode 100644 index 000000000..e48612cdb --- /dev/null +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -0,0 +1,557 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdcopybackupstask + +import ( + "context" + "fmt" + "strconv" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/features" + druidutils "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/utils/imagevector" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/go-logr/logr" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + sourcePrefix = "source-" + targetPrefix = "target-" +) + +// Reconciler reconciles EtcdCopyBackupsTask object. +type Reconciler struct { + client.Client + Config *Config + imageVector imagevector.ImageVector + logger logr.Logger +} + +// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcdcopybackupstasks,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcdcopybackupstasks/status;etcdcopybackupstasks/finalizers,verbs=get;update;patch;create + +// NewReconciler creates a new reconciler for EtcdCopyBackupsTask. +func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { + imageVector, err := utils.CreateImageVector() + if err != nil { + return nil, err + } + return NewReconcilerWithImageVector(mgr, config, imageVector), nil +} + +// NewReconcilerWithImageVector creates a new reconciler for EtcdCopyBackupsTask with an ImageVector. +// This constructor will mostly be used by tests. +func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { + return &Reconciler{ + Client: mgr.GetClient(), + Config: config, + imageVector: imageVector, + logger: log.Log.WithName("etcd-copy-backups-task-controller"), + } +} + +// Reconcile reconciles the EtcdCopyBackupsTask. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + task := &druidv1alpha1.EtcdCopyBackupsTask{} + if err := r.Get(ctx, req.NamespacedName, task); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if !task.DeletionTimestamp.IsZero() { + return r.delete(ctx, task) + } + return r.reconcile(ctx, task) +} + +func (r *Reconciler) reconcile(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (result ctrl.Result, err error) { + logger := r.logger.WithValues("etcdCopyBackupsTask", kutil.ObjectName(task), "operation", "reconcile") + + // Ensure finalizer + if !controllerutil.ContainsFinalizer(task, common.FinalizerName) { + logger.V(1).Info("Adding finalizer", "finalizerName", common.FinalizerName) + if err := controllerutils.AddFinalizers(ctx, r.Client, task, common.FinalizerName); err != nil { + return ctrl.Result{}, fmt.Errorf("could not add finalizer: %w", err) + } + } + + var status *druidv1alpha1.EtcdCopyBackupsTaskStatus + defer func() { + // Update status, on failure return the update error unless there is another error + if updateErr := r.updateStatus(ctx, task, status); updateErr != nil && err == nil { + err = fmt.Errorf("could not update status for task {name: %s, namespace: %s} : %w", task.Name, task.Namespace, updateErr) + } + }() + + // Reconcile creation or update + logger.V(1).Info("Reconciling creation or update for etcd-copy-backups-task", "name", task.Name, "namespace", task.Namespace) + if status, err = r.doReconcile(ctx, task, logger); err != nil { + return ctrl.Result{}, fmt.Errorf("could not reconcile creation or update: %w", err) + } + logger.V(1).Info("Creation or update reconciled for etcd-copy-backups-task", "name", task.Name, "namespace", task.Namespace) + + return ctrl.Result{}, nil +} + +func (r *Reconciler) doReconcile(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, logger logr.Logger) (status *druidv1alpha1.EtcdCopyBackupsTaskStatus, err error) { + status = task.Status.DeepCopy() + + var job *batchv1.Job + defer func() { + setStatusDetails(status, task.Generation, job, err) + }() + + // Get job from cluster + job, err = r.getJob(ctx, task) + if err != nil { + return status, err + } + if job != nil { + return status, nil + } + + // create a job object from task + job, err = r.createJobObject(ctx, task) + if err != nil { + return status, err + } + + // Create job + logger.Info("Creating job", "namespace", job.Namespace, "name", job.Name) + if err := r.Create(ctx, job); err != nil { + return status, fmt.Errorf("could not create job %s: %w", kutil.ObjectName(job), err) + } + + return status, nil +} + +func (r *Reconciler) delete(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (result ctrl.Result, err error) { + logger := r.logger.WithValues("task", kutil.ObjectName(task), "operation", "delete") + + // Check finalizer + if !controllerutil.ContainsFinalizer(task, common.FinalizerName) { + logger.V(1).Info("Skipping since finalizer not present", "finalizerName", common.FinalizerName) + return ctrl.Result{}, nil + } + + var status *druidv1alpha1.EtcdCopyBackupsTaskStatus + var removeFinalizer bool + defer func() { + // Only update status if the finalizer is not removed to prevent errors if the object is already gone + if !removeFinalizer { + // Update status, on failure return the update error unless there is another error + if updateErr := r.updateStatus(ctx, task, status); updateErr != nil && err == nil { + err = fmt.Errorf("could not update status: %w", updateErr) + } + } + }() + + // Reconcile deletion + logger.V(1).Info("Reconciling deletion") + if status, removeFinalizer, err = r.doDelete(ctx, task, logger); err != nil { + return ctrl.Result{}, fmt.Errorf("could not reconcile deletion: %w", err) + } + logger.V(1).Info("Deletion reconciled") + + // Remove finalizer if requested + if removeFinalizer { + logger.V(1).Info("Removing finalizer", "finalizerName", common.FinalizerName) + if err := controllerutils.RemoveFinalizers(ctx, r.Client, task, common.FinalizerName); err != nil { + return ctrl.Result{}, fmt.Errorf("could not remove finalizer: %w", err) + } + } + + return ctrl.Result{}, nil +} + +func (r *Reconciler) doDelete(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, logger logr.Logger) (status *druidv1alpha1.EtcdCopyBackupsTaskStatus, removeFinalizer bool, err error) { + status = task.Status.DeepCopy() + + var job *batchv1.Job + defer func() { + setStatusDetails(status, task.Generation, job, err) + }() + + // Get job from cluster + job, err = r.getJob(ctx, task) + if err != nil { + return status, false, err + } + if job == nil { + return status, true, nil + } + + // Delete job if needed + if job.DeletionTimestamp == nil { + logger.Info("Deleting job", "namespace", job.Namespace, "name", job.Name) + if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); client.IgnoreNotFound(err) != nil { + return status, false, fmt.Errorf("could not delete job %s: %w", kutil.ObjectName(job), err) + } + } + + return status, false, nil +} + +func (r *Reconciler) getJob(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (*batchv1.Job, error) { + job := &batchv1.Job{} + if err := r.Get(ctx, kutil.Key(task.Namespace, task.GetJobName()), job); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return job, nil +} + +func (r *Reconciler) updateStatus(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, status *druidv1alpha1.EtcdCopyBackupsTaskStatus) error { + if status == nil { + return nil + } + patch := client.MergeFromWithOptions(task.DeepCopy(), client.MergeFromWithOptimisticLock{}) + task.Status = *status + return r.Client.Status().Patch(ctx, task, patch) +} + +func setStatusDetails(status *druidv1alpha1.EtcdCopyBackupsTaskStatus, generation int64, job *batchv1.Job, err error) { + status.ObservedGeneration = &generation + if job != nil { + status.Conditions = getConditions(job.Status.Conditions) + } else { + status.Conditions = nil + } + if err != nil { + status.LastError = pointer.String(err.Error()) + } else { + status.LastError = nil + } +} + +func getConditions(jobConditions []batchv1.JobCondition) []druidv1alpha1.Condition { + var conditions []druidv1alpha1.Condition + for _, jobCondition := range jobConditions { + if conditionType := getConditionType(jobCondition.Type); conditionType != "" { + conditions = append(conditions, druidv1alpha1.Condition{ + Type: conditionType, + Status: druidv1alpha1.ConditionStatus(jobCondition.Status), + LastTransitionTime: jobCondition.LastTransitionTime, + LastUpdateTime: jobCondition.LastProbeTime, + Reason: jobCondition.Reason, + Message: jobCondition.Message, + }) + } + } + return conditions +} + +func getConditionType(jobConditionType batchv1.JobConditionType) druidv1alpha1.ConditionType { + switch jobConditionType { + case batchv1.JobComplete: + return druidv1alpha1.EtcdCopyBackupsTaskSucceeded + case batchv1.JobFailed: + return druidv1alpha1.EtcdCopyBackupsTaskFailed + } + return "" +} + +func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (*batchv1.Job, error) { + etcdBackupImage, err := druidutils.GetEtcdBackupRestoreImage(r.imageVector, r.Config.FeatureGates[features.UseEtcdWrapper]) + if err != nil { + return nil, err + } + + initContainerImage, err := druidutils.GetInitContainerImage(r.imageVector) + if err != nil { + return nil, err + } + + targetStore := task.Spec.TargetStore + targetProvider, err := druidutils.StorageProviderFromInfraProvider(targetStore.Provider) + if err != nil { + return nil, err + } + + sourceStore := task.Spec.SourceStore + sourceProvider, err := druidutils.StorageProviderFromInfraProvider(sourceStore.Provider) + if err != nil { + return nil, err + } + + // Formulate the job's arguments. + args := createJobArgs(task, sourceProvider, targetProvider) + + // Formulate the job environment variables. + env := append(createEnvVarsFromStore(&sourceStore, sourceProvider, "SOURCE_", sourcePrefix), createEnvVarsFromStore(&targetStore, targetProvider, "", "")...) + + // Formulate the job's volume mounts. + volumeMounts := append(createVolumeMountsFromStore(&sourceStore, sourceProvider, sourcePrefix, r.Config.FeatureGates[features.UseEtcdWrapper]), createVolumeMountsFromStore(&targetStore, targetProvider, targetPrefix, r.Config.FeatureGates[features.UseEtcdWrapper])...) + + // Formulate the job's volumes from the source store. + sourceVolumes, err := r.createVolumesFromStore(ctx, &sourceStore, task.Namespace, sourceProvider, sourcePrefix) + if err != nil { + return nil, err + } + + // Formulate the job's volumes from the target store. + targetVolumes, err := r.createVolumesFromStore(ctx, &targetStore, task.Namespace, targetProvider, targetPrefix) + if err != nil { + return nil, err + } + + // Combine the source and target volumes. + volumes := append(sourceVolumes, targetVolumes...) + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetJobName(), + Namespace: task.Namespace, + Annotations: map[string]string{ + common.GardenerOwnedBy: client.ObjectKeyFromObject(task).String(), + common.GardenerOwnerType: strings.ToLower(task.Kind), + }, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed, + v1beta1constants.LabelNetworkPolicyToPublicNetworks: v1beta1constants.LabelNetworkPolicyAllowed, + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: "copy-backups", + Image: *etcdBackupImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: args, + Env: env, + VolumeMounts: volumeMounts, + }, + }, + ShareProcessNamespace: pointer.Bool(true), + Volumes: volumes, + }, + }, + }, + } + + if r.Config.FeatureGates[features.UseEtcdWrapper] { + if targetProvider == druidutils.Local { + // init container to change file permissions of the folders used as store to 65532 (nonroot) + // used only with local provider + job.Spec.Template.Spec.InitContainers = []corev1.Container{ + { + Name: "change-backup-bucket-permissions", + Image: *initContainerImage, + Command: []string{"sh", "-c", "--"}, + Args: []string{fmt.Sprintf("%s%s%s%s", "chown -R 65532:65532 /home/nonroot/", *targetStore.Container, " /home/nonroot/", *sourceStore.Container)}, + VolumeMounts: volumeMounts, + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }, + } + } + job.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ + RunAsGroup: pointer.Int64(65532), + RunAsNonRoot: pointer.Bool(true), + RunAsUser: pointer.Int64(65532), + } + } + + if err := controllerutil.SetControllerReference(task, job, r.Scheme()); err != nil { + return nil, fmt.Errorf("could not set owner reference for job %v: %w", kutil.ObjectName(job), err) + } + return job, nil +} + +func createJobArgs(task *druidv1alpha1.EtcdCopyBackupsTask, sourceObjStoreProvider string, targetObjStoreProvider string) []string { + // Create the initial arguments for the copy-backups job. + args := []string{ + "copy", + "--snapstore-temp-directory=/home/nonroot/data/tmp", + } + + // Formulate the job's arguments. + args = append(args, createJobArgumentFromStore(&task.Spec.TargetStore, targetObjStoreProvider, "")...) + args = append(args, createJobArgumentFromStore(&task.Spec.SourceStore, sourceObjStoreProvider, sourcePrefix)...) + if task.Spec.MaxBackupAge != nil { + args = append(args, "--max-backup-age="+strconv.Itoa(int(*task.Spec.MaxBackupAge))) + } + + if task.Spec.MaxBackups != nil { + args = append(args, "--max-backups-to-copy="+strconv.Itoa(int(*task.Spec.MaxBackups))) + } + + if task.Spec.WaitForFinalSnapshot != nil { + args = append(args, "--wait-for-final-snapshot="+strconv.FormatBool(task.Spec.WaitForFinalSnapshot.Enabled)) + if task.Spec.WaitForFinalSnapshot.Timeout != nil { + args = append(args, "--wait-for-final-snapshot-timeout="+task.Spec.WaitForFinalSnapshot.Timeout.Duration.String()) + } + } + return args +} + +// getVolumeNamePrefix returns the appropriate volume name prefix based on the provided prefix. +// If the provided prefix is "source-", it returns the prefix; otherwise, it returns an empty string. +func getVolumeNamePrefix(prefix string) string { + if prefix == sourcePrefix { + return prefix + } + return "" +} + +// createVolumesFromStore generates a slice of VolumeMounts for an EtcdCopyBackups job based on the given StoreSpec and +// provider. The prefix is used to differentiate between source and target volume. +// This function creates the necessary Volume configurations for various storage providers. +func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1alpha1.StoreSpec, namespace, provider, prefix string) (volumes []corev1.Volume, err error) { + switch provider { + case druidutils.Local: + hostPathDirectory := corev1.HostPathDirectory + hostPathPrefix, err := druidutils.GetHostMountPathFromSecretRef(ctx, r.Client, r.logger, store, namespace) + if err != nil { + return nil, err + } + volumes = append(volumes, corev1.Volume{ + Name: prefix + "host-storage", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPathPrefix + "/" + *store.Container, + Type: &hostPathDirectory, + }, + }, + }) + case druidutils.GCS, druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + if store.SecretRef == nil { + err = fmt.Errorf("no secretRef is configured for backup %sstore", prefix) + return + } + volumes = append(volumes, corev1.Volume{ + Name: getVolumeNamePrefix(prefix) + "etcd-backup", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: store.SecretRef.Name, + }, + }, + }) + + } + return +} + +// createVolumesFromStore generates a slice of volumes for an EtcdCopyBackups job based on the given StoreSpec, namespace, +// provider, and prefix. The prefix is used to differentiate between source and target volumes. +// This function creates the necessary Volume configurations for various storage providers and returns any errors encountered. +func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volumeMountPrefix string, useEtcdWrapper bool) (volumeMounts []corev1.VolumeMount) { + switch provider { + case druidutils.Local: + if useEtcdWrapper { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: volumeMountPrefix + "host-storage", + MountPath: "/home/nonroot/" + *store.Container, + }) + } else { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: volumeMountPrefix + "host-storage", + MountPath: *store.Container, + }) + } + case druidutils.GCS: + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", + MountPath: "/var/." + getVolumeNamePrefix(volumeMountPrefix) + "gcp/", + }) + case druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", + MountPath: "/var/" + getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup/", + }) + } + return +} + +func mapToEnvVar(name, value string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + Value: value, + } +} + +// createEnvVarsFromStore generates a slice of environment variables for an EtcdCopyBackups job based on the given StoreSpec, +// storeProvider, prefix, and volumePrefix. The prefix is used to differentiate between source and target environment variables. +// This function creates the necessary environment variables for various storage providers and configurations. The generated +// environment variables include storage container information and provider-specific credentials. +func createEnvVarsFromStore(store *druidv1alpha1.StoreSpec, storeProvider, envKeyPrefix, volumePrefix string) (envVars []corev1.EnvVar) { + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.STORAGE_CONTAINER, *store.Container)) + switch storeProvider { + case druidutils.S3: + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.AWS_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) + case druidutils.ABS: + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.AZURE_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) + case druidutils.GCS: + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.GOOGLE_APPLICATION_CREDENTIALS, "/var/."+volumePrefix+"gcp/serviceaccount.json")) + case druidutils.Swift: + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.OPENSTACK_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) + case druidutils.OCS: + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.OPENSHIFT_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) + case druidutils.OSS: + envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.ALICLOUD_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) + } + return envVars +} + +// createJobArgumentFromStore generates a slice of command-line arguments for a EtcdCopyBackups job based on the given StoreSpec, +// provider, and prefix. The prefix is used to differentiate between source and target command-line arguments. +// This function is used to create the necessary command-line arguments for +// various storage providers and configurations. The generated arguments include storage provider, +// store prefix, and store container information. +func createJobArgumentFromStore(store *druidv1alpha1.StoreSpec, provider, prefix string) (arguments []string) { + if store == nil || len(provider) == 0 { + return + } + argPrefix := "--" + prefix + arguments = append(arguments, argPrefix+"storage-provider="+provider) + + if len(store.Prefix) > 0 { + arguments = append(arguments, argPrefix+"store-prefix="+store.Prefix) + } + + if store.Container != nil && len(*store.Container) > 0 { + arguments = append(arguments, argPrefix+"store-container="+*store.Container) + } + return +} diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go new file mode 100644 index 000000000..992fc11b6 --- /dev/null +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -0,0 +1,950 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdcopybackupstask + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/component-base/featuregate" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/pkg/utils" + druidutils "github.com/gardener/etcd-druid/pkg/utils" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/gardener/gardener/pkg/utils/test/matchers" +) + +var _ = Describe("EtcdCopyBackupsTaskController", func() { + + Describe("#getConditions", func() { + var ( + jobConditions []batchv1.JobCondition + ) + + BeforeEach(func() { + jobConditions = []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + { + Type: batchv1.JobFailed, + Status: corev1.ConditionFalse, + }, + } + }) + + It("should get the correct conditions from the job", func() { + conditions := getConditions(jobConditions) + Expect(len(conditions)).To(Equal(len(jobConditions))) + for i, condition := range conditions { + if condition.Type == druidv1alpha1.EtcdCopyBackupsTaskSucceeded { + Expect(jobConditions[i].Type).To(Equal(batchv1.JobComplete)) + } else if condition.Type == druidv1alpha1.EtcdCopyBackupsTaskFailed { + Expect(jobConditions[i].Type).To(Equal(batchv1.JobFailed)) + } else { + Fail("got unexpected condition type") + } + Expect(condition.Status).To(Equal(druidv1alpha1.ConditionStatus(jobConditions[i].Status))) + } + }) + }) + + Describe("#delete", func() { + var ( + ctx = context.Background() + task *druidv1alpha1.EtcdCopyBackupsTask + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + r = &Reconciler{ + Client: fakeClient, + logger: logr.Discard(), + } + ) + const ( + testTaskName = "test-etcd-backup-copy-task" + testNamespace = "test-ns" + ) + + Context("delete EtcdCopyBackupsTask object tests when it exists", func() { + BeforeEach(func() { + task = ensureEtcdCopyBackupsTaskCreation(ctx, testTaskName, testNamespace, fakeClient) + }) + AfterEach(func() { + ensureEtcdCopyBackupsTaskRemoval(ctx, testTaskName, testNamespace, fakeClient) + }) + + It("should not delete if there is no deletion timestamp set and no finalizer set", func() { + _, err := r.delete(ctx, task) + Expect(err).To(BeNil()) + foundTask := &druidv1alpha1.EtcdCopyBackupsTask{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(task), foundTask)).To(Succeed()) + Expect(client.ObjectKeyFromObject(foundTask)).To(Equal(client.ObjectKeyFromObject(task))) + }) + + It("should remove finalizer for task which does not have a corresponding job", func() { + Expect(controllerutils.AddFinalizers(ctx, fakeClient, task, common.FinalizerName)).To(Succeed()) + Expect(addDeletionTimestampToTask(ctx, task, time.Now(), fakeClient)).To(Succeed()) + + _, err := r.delete(ctx, task) + Expect(err).To(BeNil()) + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) + }).Should(BeNotFoundError()) + }) + + It("should delete job but not the task for which the deletion timestamp, finalizer is set and job is present", func() { + job := testutils.CreateEtcdCopyBackupsJob(testTaskName, testNamespace) + Expect(fakeClient.Create(ctx, job)).To(Succeed()) + Expect(controllerutils.AddFinalizers(ctx, fakeClient, task, common.FinalizerName)).To(Succeed()) + Expect(addDeletionTimestampToTask(ctx, task, time.Now(), fakeClient)).To(Succeed()) + _, err := r.delete(ctx, task) + Expect(err).To(BeNil()) + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(job), job) + }).Should(BeNotFoundError()) + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) + }).Should(BeNil()) + }) + + }) + + }) + + Describe("#createJobObject", func() { + var ( + reconciler *Reconciler + ctx = context.Background() + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + namespace = "test-ns" + unknownProvider = druidv1alpha1.StorageProvider("unknown") + ) + + BeforeEach(func() { + reconciler = &Reconciler{ + Client: fakeClient, + logger: logr.Discard(), + imageVector: imagevector.ImageVector{ + &imagevector.ImageSource{ + Name: common.BackupRestore, + Repository: "test-repo", + Tag: pointer.String("etcd-test-tag"), + }, + &imagevector.ImageSource{ + Name: common.Alpine, + Repository: "test-repo", + Tag: pointer.String("init-container-test-tag"), + }, + }, + Config: &Config{ + FeatureGates: make(map[featuregate.Feature]bool), + }, + } + }) + + DescribeTable("should create the expected job object with correct metadata, pod template, and containers for a valid input task", + func(taskName string, provider druidv1alpha1.StorageProvider, withOptionalFields bool) { + task := testutils.CreateEtcdCopyBackupsTask(taskName, namespace, provider, withOptionalFields) + errors := testutils.CreateSecrets(ctx, fakeClient, task.Namespace, task.Spec.SourceStore.SecretRef.Name, task.Spec.TargetStore.SecretRef.Name) + Expect(errors).Should(BeNil()) + + job, err := reconciler.createJobObject(ctx, task) + Expect(err).NotTo(HaveOccurred()) + Expect(job).Should(PointTo(matchJob(task, reconciler.imageVector))) + }, + Entry("with #Local provider, without optional fields", + "foo01", druidv1alpha1.StorageProvider("Local"), false), + Entry("with #Local provider, with optional fields", + "foo02", druidv1alpha1.StorageProvider("Local"), true), + Entry("with #S3 storage provider, without optional fields", + "foo03", druidv1alpha1.StorageProvider("aws"), false), + Entry("with #S3 storage provider, with optional fields", + "foo04", druidv1alpha1.StorageProvider("aws"), true), + Entry("with #AZURE storage provider, without optional fields", + "foo05", druidv1alpha1.StorageProvider("azure"), false), + Entry("with #AZURE storage provider, with optional fields", + "foo06", druidv1alpha1.StorageProvider("azure"), true), + Entry("with #GCP storage provider, without optional fields", + "foo07", druidv1alpha1.StorageProvider("gcp"), false), + Entry("with #GCP storage provider, with optional fields", + "foo08", druidv1alpha1.StorageProvider("gcp"), true), + Entry("with #OPENSTACK storage provider, without optional fields", + "foo09", druidv1alpha1.StorageProvider("openstack"), false), + Entry("with #OPENSTACK storage provider, with optional fields", + "foo10", druidv1alpha1.StorageProvider("openstack"), true), + Entry("with #ALICLOUD storage provider, without optional fields", + "foo11", druidv1alpha1.StorageProvider("alicloud"), false), + Entry("with #ALICLOUD storage provider, with optional fields", + "foo12", druidv1alpha1.StorageProvider("alicloud"), true), + ) + + Context("when etcd-backup image is not found", func() { + It("should return error", func() { + reconciler.imageVector = nil + task := testutils.CreateEtcdCopyBackupsTask("test", namespace, "Local", true) + job, err := reconciler.createJobObject(ctx, task) + + Expect(job).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when target store provider is unknown", func() { + It("should return error", func() { + task := testutils.CreateEtcdCopyBackupsTask("test", namespace, "Local", true) + task.Spec.TargetStore.Provider = &unknownProvider + job, err := reconciler.createJobObject(ctx, task) + + Expect(job).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when source store provider is unknown", func() { + It("should return error", func() { + task := testutils.CreateEtcdCopyBackupsTask("test", namespace, "Local", true) + task.Spec.SourceStore.Provider = &unknownProvider + job, err := reconciler.createJobObject(ctx, task) + + Expect(job).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Describe("#createJobArgumentFromStore", func() { + Context("when given a nil store", func() { + It("returns an empty argument slice", func() { + Expect(createJobArgumentFromStore(nil, "provider", "prefix")).To(BeEmpty()) + }) + }) + + Context("when given a empty provider", func() { + It("returns an empty argument slice", func() { + Expect(createJobArgumentFromStore(nil, "", "prefix")).To(BeEmpty()) + }) + }) + + Context("when given a non-nil store", func() { + var ( + store druidv1alpha1.StoreSpec + provider string + prefix string + ) + + BeforeEach(func() { + store = druidv1alpha1.StoreSpec{ + Prefix: "store_prefix", + Container: pointer.String("store_container"), + } + provider = "storage_provider" + prefix = "prefix" + }) + + It("returns a argument slice with provider, prefix, and container information", func() { + expected := []string{ + "--prefixstorage-provider=storage_provider", + "--prefixstore-prefix=store_prefix", + "--prefixstore-container=store_container", + } + Expect(createJobArgumentFromStore(&store, provider, prefix)).To(Equal(expected)) + }) + + It("should return a argument slice with provider and prefix information only when StoreSpec.Container is nil", func() { + expected := []string{ + "--prefixstorage-provider=storage_provider", + "--prefixstore-prefix=store_prefix", + } + store.Container = nil + Expect(createJobArgumentFromStore(&store, provider, prefix)).To(Equal(expected)) + }) + + It("should return a argument slice with provider and container information only when StoreSpec.Prefix is empty", func() { + expected := []string{ + "--prefixstorage-provider=storage_provider", + "--prefixstore-container=store_container", + } + store.Prefix = "" + Expect(createJobArgumentFromStore(&store, provider, prefix)).To(Equal(expected)) + }) + + It("returns an empty argument slice when StoreSpec.Provider is empty", func() { + provider = "" + Expect(createJobArgumentFromStore(&store, provider, prefix)).To(BeEmpty()) + }) + }) + }) + + Describe("#createJobArguments", func() { + var ( + providerLocal = druidv1alpha1.StorageProvider(druidutils.Local) + providerS3 = druidv1alpha1.StorageProvider(druidutils.S3) + task *druidv1alpha1.EtcdCopyBackupsTask + expected = []string{ + "copy", + "--snapstore-temp-directory=/home/nonroot/data/tmp", + "--storage-provider=S3", + "--store-prefix=/target", + "--store-container=target-container", + "--source-storage-provider=Local", + "--source-store-prefix=/source", + "--source-store-container=source-container", + } + ) + + BeforeEach(func() { + task = &druidv1alpha1.EtcdCopyBackupsTask{ + Spec: druidv1alpha1.EtcdCopyBackupsTaskSpec{ + SourceStore: druidv1alpha1.StoreSpec{ + Prefix: "/source", + Container: pointer.String("source-container"), + Provider: &providerLocal, + }, + TargetStore: druidv1alpha1.StoreSpec{ + Prefix: "/target", + Container: pointer.String("target-container"), + Provider: &providerS3, + SecretRef: &corev1.SecretReference{ + Name: "test-secret", + }, + }, + }, + } + }) + + It("should create the correct arguments", func() { + arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + Expect(arguments).To(Equal(expected)) + }) + + It("should include the max backup age in the arguments", func() { + task.Spec.MaxBackupAge = pointer.Uint32(10) + arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + Expect(arguments).To(Equal(append(expected, "--max-backup-age=10"))) + }) + + It("should include the max number of backups in the arguments", func() { + task.Spec.MaxBackups = pointer.Uint32(5) + arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + Expect(arguments).To(Equal(append(expected, "--max-backups-to-copy=5"))) + }) + + It("should include the wait for final snapshot in the arguments", func() { + task.Spec.WaitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ + Enabled: true, + } + arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true"))) + }) + + It("should include the wait for final snapshot and timeout in the arguments", func() { + task.Spec.WaitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ + Enabled: true, + Timeout: &metav1.Duration{Duration: time.Minute}, + } + arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true", "--wait-for-final-snapshot-timeout=1m0s"))) + }) + }) + + Describe("#createEnvVarsFromStore", func() { + var ( + envKeyPrefix = "SOURCE_" + volumePrefix = "source-" + container = "source-container" + storeSpec *druidv1alpha1.StoreSpec + ) + // Loop through different storage providers to test with + for _, p := range []string{ + druidutils.ABS, + druidutils.GCS, + druidutils.S3, + druidutils.Swift, + druidutils.OSS, + druidutils.OCS, + } { + Context(fmt.Sprintf("with provider #%s", p), func() { + provider := p + BeforeEach(func() { + storageProvider := druidv1alpha1.StorageProvider(provider) + storeSpec = &druidv1alpha1.StoreSpec{ + Container: &container, + Provider: &storageProvider, + } + }) + + It("should create the correct env vars", func() { + envVars := createEnvVarsFromStore(storeSpec, provider, envKeyPrefix, volumePrefix) + checkEnvVars(envVars, provider, container, envKeyPrefix, volumePrefix) + + }) + }) + } + Context("with provider #Local", func() { + BeforeEach(func() { + storageProvider := druidv1alpha1.StorageProvider(druidutils.Local) + storeSpec = &druidv1alpha1.StoreSpec{ + Container: &container, + Provider: &storageProvider, + } + }) + + It("should create the correct env vars", func() { + envVars := createEnvVarsFromStore(storeSpec, druidutils.Local, envKeyPrefix, volumePrefix) + checkEnvVars(envVars, druidutils.Local, container, envKeyPrefix, volumePrefix) + + }) + }) + }) + + Describe("#createVolumeMountsFromStore", func() { + var ( + volumeMountPrefix = "source-" + storeSpec *druidv1alpha1.StoreSpec + ) + // Loop through different storage providers to test with + for _, p := range []string{ + druidutils.Local, + druidutils.ABS, + druidutils.GCS, + druidutils.S3, + druidutils.Swift, + druidutils.OSS, + druidutils.OCS, + } { + Context(fmt.Sprintf("with provider #%s", p), func() { + provider := p + BeforeEach(func() { + storageProvider := druidv1alpha1.StorageProvider(provider) + storeSpec = &druidv1alpha1.StoreSpec{ + Container: pointer.String("source-container"), + Provider: &storageProvider, + } + }) + + It("should create the correct volume mounts", func() { + volumeMounts := createVolumeMountsFromStore(storeSpec, provider, volumeMountPrefix, false) + Expect(volumeMounts).To(HaveLen(1)) + + expectedMountPath := "" + expectedMountName := "" + + switch provider { + case druidutils.Local: + expectedMountName = volumeMountPrefix + "host-storage" + expectedMountPath = *storeSpec.Container + case druidutils.GCS: + expectedMountName = volumeMountPrefix + "etcd-backup" + expectedMountPath = "/var/." + volumeMountPrefix + "gcp/" + case druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + expectedMountName = volumeMountPrefix + "etcd-backup" + expectedMountPath = "/var/" + volumeMountPrefix + "etcd-backup/" + default: + Fail(fmt.Sprintf("Unknown provider: %s", provider)) + } + + Expect(volumeMounts[0].Name).To(Equal(expectedMountName)) + Expect(volumeMounts[0].MountPath).To(Equal(expectedMountPath)) + }) + }) + } + }) + + Describe("#createVolumesFromStore", func() { + Context("with provider #Local", func() { + var ( + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + ctx = context.Background() + secret *corev1.Secret + providerLocal = druidv1alpha1.StorageProvider("Local") + namespace = "test-ns" + reconciler = &Reconciler{ + Client: fakeClient, + logger: logr.Discard(), + } + + store = &druidv1alpha1.StoreSpec{ + Container: pointer.String("source-container"), + Prefix: "/tmp", + Provider: &providerLocal, + } + ) + + BeforeEach(func() { + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: namespace, + }, + } + }) + + AfterEach(func() { + Expect(fakeClient.Delete(ctx, secret)).To(Succeed()) + }) + + It("should create the correct volumes when secret data hostPath is set", func() { + secret.Data = map[string][]byte{ + druidutils.EtcdBackupSecretHostPath: []byte("/test/hostPath"), + } + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + store.SecretRef = &corev1.SecretReference{Name: secret.Name} + volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(providerLocal), "source-") + Expect(err).NotTo(HaveOccurred()) + Expect(volumes).To(HaveLen(1)) + Expect(volumes[0].Name).To(Equal("source-host-storage")) + + hostPathVolumeSource := volumes[0].VolumeSource.HostPath + Expect(hostPathVolumeSource).NotTo(BeNil()) + Expect(hostPathVolumeSource.Path).To(Equal("/test/hostPath/" + *store.Container)) + Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) + }) + + It("should create the correct volumes when secret data hostPath is not set", func() { + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + store.SecretRef = &corev1.SecretReference{Name: secret.Name} + volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(providerLocal), "source-") + Expect(err).NotTo(HaveOccurred()) + Expect(volumes).To(HaveLen(1)) + Expect(volumes[0].Name).To(Equal("source-host-storage")) + + hostPathVolumeSource := volumes[0].VolumeSource.HostPath + Expect(hostPathVolumeSource).NotTo(BeNil()) + Expect(hostPathVolumeSource.Path).To(Equal(druidutils.LocalProviderDefaultMountPath + "/" + *store.Container)) + Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) + }) + + It("should create the correct volumes when store.SecretRef is not referred", func() { + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + store.SecretRef = &corev1.SecretReference{Name: secret.Name} + volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(providerLocal), "source-") + Expect(err).NotTo(HaveOccurred()) + Expect(volumes).To(HaveLen(1)) + Expect(volumes[0].Name).To(Equal("source-host-storage")) + + hostPathVolumeSource := volumes[0].VolumeSource.HostPath + Expect(hostPathVolumeSource).NotTo(BeNil()) + Expect(hostPathVolumeSource.Path).To(Equal(druidutils.LocalProviderDefaultMountPath + "/" + *store.Container)) + Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) + }) + }) + + Context("with provider", func() { + var ( + storageProvider druidv1alpha1.StorageProvider + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + ctx = context.Background() + secret *corev1.Secret + store *druidv1alpha1.StoreSpec + namespace = "test-ns" + reconciler = &Reconciler{ + Client: fakeClient, + logger: logr.Discard(), + } + ) + + // Loop through different storage providers to test with + for _, p := range []string{ + druidutils.ABS, + druidutils.GCS, + druidutils.S3, + druidutils.Swift, + druidutils.OSS, + druidutils.OCS, + } { + Context(fmt.Sprintf("#%s", p), func() { + BeforeEach(func() { + provider := p + // Set up test variables and create necessary secrets + storageProvider = druidv1alpha1.StorageProvider(provider) + store = &druidv1alpha1.StoreSpec{ + Container: pointer.String("source-container"), + Provider: &storageProvider, + } + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret-" + provider, + Namespace: namespace, + }, + } + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + }) + + AfterEach(func() { + // Clean up secret after each test case + Expect(fakeClient.Delete(ctx, secret)).To(Succeed()) + }) + + It("should create the correct volumes", func() { + // Call the function being tested with a valid secret reference + store.SecretRef = &corev1.SecretReference{Name: secret.Name} + volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(storageProvider), "source-") + Expect(err).NotTo(HaveOccurred()) + Expect(volumes).To(HaveLen(1)) + Expect(volumes[0].Name).To(Equal("source-etcd-backup")) + + // Assert that the volume is created correctly with the expected secret + volumeSource := volumes[0].VolumeSource + Expect(volumeSource).NotTo(BeNil()) + Expect(volumeSource.Secret).NotTo(BeNil()) + Expect(*volumeSource.Secret).To(Equal(corev1.SecretVolumeSource{ + SecretName: store.SecretRef.Name, + })) + }) + + It("should return an error when secret reference is invalid", func() { + // Call the function being tested with an invalid secret reference + volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(storageProvider), "source-") + + // Assert that an error is returned and no volumes are created + Expect(err.Error()).To(Equal("no secretRef is configured for backup source-store")) + Expect(volumes).To(HaveLen(0)) + }) + }) + } + }) + }) + +}) + +func ensureEtcdCopyBackupsTaskCreation(ctx context.Context, name, namespace string, fakeClient client.WithWatch) *druidv1alpha1.EtcdCopyBackupsTask { + task := testutils.CreateEtcdCopyBackupsTask(name, namespace, "aws", false) + By("create task") + Expect(fakeClient.Create(ctx, task)).To(Succeed()) + + By("Ensure that copy backups task is created") + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) + }).Should(Succeed()) + + return task +} + +func ensureEtcdCopyBackupsTaskRemoval(ctx context.Context, name, namespace string, fakeClient client.WithWatch) { + task := &druidv1alpha1.EtcdCopyBackupsTask{} + if err := fakeClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, task); err != nil { + Expect(err).To(BeNotFoundError()) + return + } + + By("Remove any existing finalizers on EtcdCopyBackupsTask") + Expect(controllerutils.RemoveAllFinalizers(ctx, fakeClient, task)).To(Succeed()) + + By("Delete EtcdCopyBackupsTask") + err := fakeClient.Delete(ctx, task) + if err != nil { + Expect(err).Should(BeNotFoundError()) + } + + By("Ensure EtcdCopyBackupsTask is deleted") + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) + }).Should(BeNotFoundError()) +} + +func addDeletionTimestampToTask(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, deletionTime time.Time, fakeClient client.WithWatch) error { + patch := client.MergeFrom(task.DeepCopy()) + task.DeletionTimestamp = &metav1.Time{Time: deletionTime} + return fakeClient.Patch(ctx, task, patch) +} + +func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefix, volumePrefix string) { + expected := []corev1.EnvVar{ + { + Name: envKeyPrefix + "STORAGE_CONTAINER", + Value: container, + }} + mapToEnvVarKey := map[string]string{ + druidutils.S3: envKeyPrefix + common.AWS_APPLICATION_CREDENTIALS, + druidutils.ABS: envKeyPrefix + common.AZURE_APPLICATION_CREDENTIALS, + druidutils.GCS: envKeyPrefix + common.GOOGLE_APPLICATION_CREDENTIALS, + druidutils.Swift: envKeyPrefix + common.OPENSTACK_APPLICATION_CREDENTIALS, + druidutils.OCS: envKeyPrefix + common.OPENSHIFT_APPLICATION_CREDENTIALS, + druidutils.OSS: envKeyPrefix + common.ALICLOUD_APPLICATION_CREDENTIALS, + } + switch storeProvider { + case druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + expected = append(expected, corev1.EnvVar{ + Name: mapToEnvVarKey[storeProvider], + Value: "/var/" + volumePrefix + "etcd-backup", + }) + case druidutils.GCS: + expected = append(expected, corev1.EnvVar{ + Name: mapToEnvVarKey[storeProvider], + Value: "/var/." + volumePrefix + "gcp/serviceaccount.json", + }) + } + Expect(envVars).To(Equal(expected)) +} + +func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.ImageVector) gomegatypes.GomegaMatcher { + sourceProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.SourceStore.Provider) + Expect(err).NotTo(HaveOccurred()) + targetProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) + Expect(err).NotTo(HaveOccurred()) + + images, err := imagevector.FindImages(imageVector, []string{common.BackupRestore}) + Expect(err).NotTo(HaveOccurred()) + backupRestoreImage := images[common.BackupRestore] + + matcher := MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(task.Name + "-worker"), + "Namespace": Equal(task.Namespace), + "Annotations": MatchKeys(IgnoreExtras, Keys{ + "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", task.Namespace, task.Name)), + "gardener.cloud/owner-type": Equal("etcdcopybackupstask"), + }), + "OwnerReferences": MatchAllElements(testutils.OwnerRefIterator, Elements{ + task.Name: MatchAllFields(Fields{ + "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), + "Kind": Equal("EtcdCopyBackupsTask"), + "Name": Equal(task.Name), + "UID": Equal(task.UID), + "Controller": PointTo(Equal(true)), + "BlockOwnerDeletion": PointTo(Equal(true)), + }), + }), + }), + "Spec": MatchFields(IgnoreExtras, Fields{ + "Template": MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Labels": MatchKeys(IgnoreExtras, Keys{ + "networking.gardener.cloud/to-dns": Equal("allowed"), + "networking.gardener.cloud/to-public-networks": Equal("allowed"), + }), + }), + "Spec": MatchFields(IgnoreExtras, Fields{ + "RestartPolicy": Equal(corev1.RestartPolicyOnFailure), + "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ + "copy-backups": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("copy-backups"), + "Image": Equal(fmt.Sprintf("%s:%s", backupRestoreImage.Repository, *backupRestoreImage.Tag)), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + "Args": MatchAllElements(testutils.CmdIterator, getArgElements(task, sourceProvider, targetProvider)), + "Env": MatchElements(testutils.EnvIterator, IgnoreExtras, getEnvElements(task)), + }), + }), + }), + }), + }), + }) + + return And(matcher, matchJobWithProviders(task, sourceProvider, targetProvider)) +} + +func getArgElements(task *druidv1alpha1.EtcdCopyBackupsTask, sourceProvider, targetProvider string) Elements { + elements := Elements{ + "copy": Equal("copy"), + "--snapstore-temp-directory=/home/nonroot/data/tmp": Equal("--snapstore-temp-directory=/home/nonroot/data/tmp"), + } + if targetProvider != "" { + addEqual(elements, fmt.Sprintf("%s=%s", "--storage-provider", targetProvider)) + } + if task.Spec.TargetStore.Prefix != "" { + addEqual(elements, fmt.Sprintf("%s=%s", "--store-prefix", task.Spec.TargetStore.Prefix)) + } + if task.Spec.TargetStore.Container != nil && *task.Spec.TargetStore.Container != "" { + addEqual(elements, fmt.Sprintf("%s=%s", "--store-container", *task.Spec.TargetStore.Container)) + } + if sourceProvider != "" { + addEqual(elements, fmt.Sprintf("%s=%s", "--source-storage-provider", sourceProvider)) + } + if task.Spec.SourceStore.Prefix != "" { + addEqual(elements, fmt.Sprintf("%s=%s", "--source-store-prefix", task.Spec.SourceStore.Prefix)) + } + if task.Spec.SourceStore.Container != nil && *task.Spec.SourceStore.Container != "" { + addEqual(elements, fmt.Sprintf("%s=%s", "--source-store-container", *task.Spec.SourceStore.Container)) + } + if task.Spec.MaxBackupAge != nil && *task.Spec.MaxBackupAge != 0 { + addEqual(elements, fmt.Sprintf("%s=%d", "--max-backup-age", *task.Spec.MaxBackupAge)) + } + if task.Spec.MaxBackups != nil && *task.Spec.MaxBackups != 0 { + addEqual(elements, fmt.Sprintf("%s=%d", "--max-backups-to-copy", *task.Spec.MaxBackups)) + } + if task.Spec.WaitForFinalSnapshot != nil && task.Spec.WaitForFinalSnapshot.Enabled { + addEqual(elements, fmt.Sprintf("%s=%t", "--wait-for-final-snapshot", task.Spec.WaitForFinalSnapshot.Enabled)) + if task.Spec.WaitForFinalSnapshot.Timeout != nil && task.Spec.WaitForFinalSnapshot.Timeout.Duration != 0 { + addEqual(elements, fmt.Sprintf("%s=%s", "--wait-for-final-snapshot-timeout", task.Spec.WaitForFinalSnapshot.Timeout.Duration.String())) + } + } + return elements +} + +func getEnvElements(task *druidv1alpha1.EtcdCopyBackupsTask) Elements { + elements := Elements{} + if task.Spec.TargetStore.Container != nil && *task.Spec.TargetStore.Container != "" { + elements["STORAGE_CONTAINER"] = MatchFields(IgnoreExtras, Fields{ + "Name": Equal("STORAGE_CONTAINER"), + "Value": Equal(*task.Spec.TargetStore.Container), + }) + } + if task.Spec.SourceStore.Container != nil && *task.Spec.SourceStore.Container != "" { + elements["SOURCE_STORAGE_CONTAINER"] = MatchFields(IgnoreExtras, Fields{ + "Name": Equal("SOURCE_STORAGE_CONTAINER"), + "Value": Equal(*task.Spec.SourceStore.Container), + }) + } + return elements +} + +func matchJobWithProviders(task *druidv1alpha1.EtcdCopyBackupsTask, sourceProvider, targetProvider string) gomegatypes.GomegaMatcher { + matcher := MatchFields(IgnoreExtras, Fields{ + "Spec": MatchFields(IgnoreExtras, Fields{ + "Template": MatchFields(IgnoreExtras, Fields{ + "Spec": MatchFields(IgnoreExtras, Fields{ + "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ + "copy-backups": MatchFields(IgnoreExtras, Fields{ + "Env": And( + MatchElements(testutils.EnvIterator, IgnoreExtras, getProviderEnvElements(targetProvider, "", "")), + MatchElements(testutils.EnvIterator, IgnoreExtras, getProviderEnvElements(sourceProvider, "SOURCE_", "source-")), + ), + }), + }), + }), + }), + }), + }) + if sourceProvider == "GCS" || targetProvider == "GCS" { + volumeMatcher := MatchFields(IgnoreExtras, Fields{ + "Spec": MatchFields(IgnoreExtras, Fields{ + "Template": MatchFields(IgnoreExtras, Fields{ + "Spec": MatchFields(IgnoreExtras, Fields{ + "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ + "copy-backups": MatchFields(IgnoreExtras, Fields{ + "VolumeMounts": And( + MatchElements(testutils.VolumeMountIterator, IgnoreExtras, getVolumeMountsElements(targetProvider, "")), + MatchElements(testutils.VolumeMountIterator, IgnoreExtras, getVolumeMountsElements(sourceProvider, "source-")), + ), + }), + }), + "Volumes": And( + MatchElements(testutils.VolumeIterator, IgnoreExtras, getVolumesElements("", &task.Spec.TargetStore)), + MatchElements(testutils.VolumeIterator, IgnoreExtras, getVolumesElements("source-", &task.Spec.SourceStore)), + ), + }), + }), + }), + }) + return And(matcher, volumeMatcher) + } + return matcher +} + +func getProviderEnvElements(storeProvider, prefix, volumePrefix string) Elements { + switch storeProvider { + case "S3": + return Elements{ + prefix + "AWS_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + "AWS_APPLICATION_CREDENTIALS"), + "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), + }), + } + case "ABS": + return Elements{ + prefix + "AZURE_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + "AZURE_APPLICATION_CREDENTIALS"), + "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), + }), + } + case "GCS": + return Elements{ + prefix + "GOOGLE_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + "GOOGLE_APPLICATION_CREDENTIALS"), + "Value": Equal(fmt.Sprintf("/var/.%sgcp/serviceaccount.json", volumePrefix)), + }), + } + case "Swift": + return Elements{ + prefix + "OPENSTACK_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + "OPENSTACK_APPLICATION_CREDENTIALS"), + "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), + }), + } + case "OSS": + return Elements{ + prefix + "ALICLOUD_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + "ALICLOUD_APPLICATION_CREDENTIALS"), + "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), + }), + } + case "OCS": + return Elements{ + prefix + "OPENSHIFT_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + "OPENSHIFT_APPLICATION_CREDENTIALS"), + "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), + }), + } + default: + return nil + } +} + +func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { + switch storeProvider { + case "GCS": + return Elements{ + volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + "etcd-backup"), + "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), + }), + } + default: + return Elements{ + volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + "etcd-backup"), + "MountPath": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), + }), + } + } +} + +func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Elements { + return Elements{ + volumePrefix + "etcd-backup": MatchAllFields(Fields{ + "Name": Equal(volumePrefix + "etcd-backup"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(store.SecretRef.Name), + })), + }), + }), + } +} + +func addEqual(elements Elements, s string) { + elements[s] = Equal(s) +} diff --git a/internal/controller/etcdcopybackupstask/register.go b/internal/controller/etcdcopybackupstask/register.go new file mode 100644 index 000000000..30307ad12 --- /dev/null +++ b/internal/controller/etcdcopybackupstask/register.go @@ -0,0 +1,42 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdcopybackupstask + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + batchv1 "k8s.io/api/batch/v1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlbuilder "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +const controllerName = "etcdcopybackupstask-controller" + +// RegisterWithManager registers the EtcdCopyBackupsTask Controller with the given controller manager. +func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.Config.Workers, + }). + For( + &druidv1alpha1.EtcdCopyBackupsTask{}, + ctrlbuilder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). + Owns(&batchv1.Job{}). + Complete(r) +} diff --git a/internal/controller/manager.go b/internal/controller/manager.go new file mode 100644 index 000000000..c6d9447cd --- /dev/null +++ b/internal/controller/manager.go @@ -0,0 +1,130 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "time" + + "github.com/gardener/etcd-druid/controllers/custodian" + "github.com/gardener/etcd-druid/internal/client/kubernetes" + "github.com/gardener/etcd-druid/internal/controller/compaction" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" + "github.com/gardener/etcd-druid/internal/controller/secret" + coordinationv1 "k8s.io/api/coordination/v1" + coordinationv1beta1 "k8s.io/api/coordination/v1beta1" + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + eventsv1beta1 "k8s.io/api/events/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + defaultTimeout = time.Minute +) + +// CreateManagerWithControllers creates a controller manager and adds all the controllers to the controller-manager using the passed in ManagerConfig. +func CreateManagerWithControllers(config *ManagerConfig) (ctrl.Manager, error) { + var ( + err error + mgr ctrl.Manager + ) + + config.populateControllersFeatureGates() + + if mgr, err = createManager(config); err != nil { + return nil, err + } + if err = registerControllersWithManager(mgr, config); err != nil { + return nil, err + } + + return mgr, nil +} + +func createManager(config *ManagerConfig) (ctrl.Manager, error) { + // TODO: this can be removed once we have an improved informer, see https://github.com/gardener/etcd-druid/issues/215 + // list of objects which should not be cached. + uncachedObjects := []client.Object{ + &corev1.Event{}, + &eventsv1beta1.Event{}, + &eventsv1.Event{}, + } + + if config.DisableLeaseCache { + uncachedObjects = append(uncachedObjects, &coordinationv1.Lease{}, &coordinationv1beta1.Lease{}) + } + + return ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + ClientDisableCacheFor: uncachedObjects, + Scheme: kubernetes.Scheme, + MetricsBindAddress: config.MetricsAddr, + LeaderElection: config.EnableLeaderElection, + LeaderElectionID: config.LeaderElectionID, + LeaderElectionResourceLock: config.LeaderElectionResourceLock, + }) +} + +func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) error { + var err error + + // Add etcd reconciler to the manager + etcdReconciler, err := etcd.NewReconciler(mgr, config.EtcdControllerConfig) + if err != nil { + return err + } + if err = etcdReconciler.RegisterWithManager(mgr); err != nil { + return err + } + + // Add custodian reconciler to the manager + custodianReconciler := custodian.NewReconciler(mgr, config.CustodianControllerConfig) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + if err = custodianReconciler.RegisterWithManager(ctx, mgr, config.IgnoreOperationAnnotation); err != nil { + return err + } + + // Add compaction reconciler to the manager if the CLI flag enable-backup-compaction is true. + if config.CompactionControllerConfig.EnableBackupCompaction { + compactionReconciler, err := compaction.NewReconciler(mgr, config.CompactionControllerConfig) + if err != nil { + return err + } + if err = compactionReconciler.RegisterWithManager(mgr); err != nil { + return err + } + } + + // Add etcd-copy-backups-task reconciler to the manager + etcdCopyBackupsTaskReconciler, err := etcdcopybackupstask.NewReconciler(mgr, config.EtcdCopyBackupsTaskControllerConfig) + if err != nil { + return err + } + if err = etcdCopyBackupsTaskReconciler.RegisterWithManager(mgr); err != nil { + return err + } + + // Add secret reconciler to the manager + return secret.NewReconciler( + mgr, + config.SecretControllerConfig, + ).RegisterWithManager(ctx, mgr) +} diff --git a/internal/controller/predicate/predicate.go b/internal/controller/predicate/predicate.go new file mode 100644 index 000000000..260f69587 --- /dev/null +++ b/internal/controller/predicate/predicate.go @@ -0,0 +1,221 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicate + +import ( + "reflect" + "strings" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +) + +func hasOperationAnnotation(obj client.Object) bool { + return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile +} + +// HasOperationAnnotation is a predicate for the operation annotation. +func HasOperationAnnotation() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return hasOperationAnnotation(event.Object) + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return hasOperationAnnotation(event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return hasOperationAnnotation(event.Object) + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return true + }, + } +} + +// LastOperationNotSuccessful is a predicate for unsuccessful last operations for creation events. +func LastOperationNotSuccessful() predicate.Predicate { + operationNotSucceeded := func(obj runtime.Object) bool { + etcd, ok := obj.(*druidv1alpha1.Etcd) + if !ok { + return false + } + if etcd.Status.LastError != nil { + return true + } + return false + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return operationNotSucceeded(event.Object) + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return operationNotSucceeded(event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return operationNotSucceeded(event.Object) + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return operationNotSucceeded(event.Object) + }, + } +} + +// StatefulSetStatusChange is a predicate for status changes of `StatefulSet` resources. +func StatefulSetStatusChange() predicate.Predicate { + statusChange := func(objOld, objNew client.Object) bool { + stsOld, ok := objOld.(*appsv1.StatefulSet) + if !ok { + return false + } + stsNew, ok := objNew.(*appsv1.StatefulSet) + if !ok { + return false + } + return !apiequality.Semantic.DeepEqual(stsOld.Status, stsNew.Status) + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return true + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return statusChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return true + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return true + }, + } +} + +// EtcdReconciliationFinished is a predicate to use for etcd resources whose reconciliation has finished. +func EtcdReconciliationFinished(ignoreOperationAnnotation bool) predicate.Predicate { + reconciliationFinished := func(obj client.Object) bool { + etcd, ok := obj.(*druidv1alpha1.Etcd) + if !ok { + return false + } + + if etcd.Status.ObservedGeneration == nil { + return false + } + + condition := *etcd.Status.ObservedGeneration == etcd.Generation + + if !ignoreOperationAnnotation { + condition = condition && !hasOperationAnnotation(etcd) + } + + return condition + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return reconciliationFinished(event.Object) + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return reconciliationFinished(event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return reconciliationFinished(event.Object) + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return false + }, + } +} + +// SnapshotRevisionChanged is a predicate that is `true` if the passed lease object is a snapshot lease and if the lease +// object's holderIdentity is updated. +func SnapshotRevisionChanged() predicate.Predicate { + isSnapshotLease := func(obj client.Object) bool { + lease, ok := obj.(*coordinationv1.Lease) + if !ok { + return false + } + + return strings.HasSuffix(lease.Name, "full-snap") || strings.HasSuffix(lease.Name, "delta-snap") + } + + holderIdentityChange := func(objOld, objNew client.Object) bool { + leaseOld, ok := objOld.(*coordinationv1.Lease) + if !ok { + return false + } + leaseNew, ok := objNew.(*coordinationv1.Lease) + if !ok { + return false + } + + return !reflect.DeepEqual(leaseOld.Spec.HolderIdentity, leaseNew.Spec.HolderIdentity) + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return isSnapshotLease(event.Object) + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return isSnapshotLease(event.ObjectNew) && holderIdentityChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return false + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return false + }, + } +} + +// JobStatusChanged is a predicate that is `true` if the status of a job changes. +func JobStatusChanged() predicate.Predicate { + statusChange := func(objOld, objNew client.Object) bool { + jobOld, ok := objOld.(*batchv1.Job) + if !ok { + return false + } + jobNew, ok := objNew.(*batchv1.Job) + if !ok { + return false + } + return !apiequality.Semantic.DeepEqual(jobOld.Status, jobNew.Status) + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return false + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return statusChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return false + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return false + }, + } +} diff --git a/internal/controller/predicate/predicate_suite_test.go b/internal/controller/predicate/predicate_suite_test.go new file mode 100644 index 000000000..0829a0f8e --- /dev/null +++ b/internal/controller/predicate/predicate_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicate_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPredicate(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Predicate Suite") +} diff --git a/internal/controller/predicate/predicate_test.go b/internal/controller/predicate/predicate_test.go new file mode 100644 index 000000000..e4cf87fe0 --- /dev/null +++ b/internal/controller/predicate/predicate_test.go @@ -0,0 +1,668 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicate_test + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/controllers/predicate" + + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +var _ = Describe("Druid Predicate", func() { + var ( + obj, oldObj client.Object + + createEvent event.CreateEvent + updateEvent event.UpdateEvent + deleteEvent event.DeleteEvent + genericEvent event.GenericEvent + ) + + JustBeforeEach(func() { + createEvent = event.CreateEvent{ + Object: obj, + } + updateEvent = event.UpdateEvent{ + ObjectOld: oldObj, + ObjectNew: obj, + } + deleteEvent = event.DeleteEvent{ + Object: obj, + } + genericEvent = event.GenericEvent{ + Object: obj, + } + }) + + Describe("#StatefulSet", func() { + var pred predicate.Predicate + + JustBeforeEach(func() { + pred = StatefulSetStatusChange() + }) + + Context("when status matches", func() { + BeforeEach(func() { + obj = &appsv1.StatefulSet{ + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + }, + } + oldObj = &appsv1.StatefulSet{ + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + + Context("when status differs", func() { + BeforeEach(func() { + obj = &appsv1.StatefulSet{ + Status: appsv1.StatefulSetStatus{ + Replicas: 2, + }, + } + oldObj = &appsv1.StatefulSet{ + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + }) + + Describe("#Lease", func() { + var pred predicate.Predicate + + JustBeforeEach(func() { + pred = SnapshotRevisionChanged() + }) + + Context("when holder identity is nil for delta snap leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-delta-snap", + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-delta-snap", + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when holder identity matches for delta snap leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-delta-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-delta-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when holder identity differs for delta snap leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-delta-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("5"), + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-delta-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when holder identity is nil for full snap leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-full-snap", + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-full-snap", + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when holder identity matches for full snap leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-full-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-full-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when holder identity differs for full snap leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-full-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("5"), + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-full-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when holder identity differs for any other leases", func() { + BeforeEach(func() { + obj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("5"), + }, + } + oldObj = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("0"), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + }) + + Describe("#Job", func() { + var pred predicate.Predicate + + JustBeforeEach(func() { + pred = JobStatusChanged() + }) + + Context("when status matches", func() { + BeforeEach(func() { + now := metav1.Now() + obj = &batchv1.Job{ + Status: batchv1.JobStatus{ + CompletionTime: &now, + }, + } + oldObj = &batchv1.Job{ + Status: batchv1.JobStatus{ + CompletionTime: &now, + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when status differs", func() { + BeforeEach(func() { + now := metav1.Now() + obj = &batchv1.Job{ + Status: batchv1.JobStatus{ + CompletionTime: nil, + }, + } + oldObj = &batchv1.Job{ + Status: batchv1.JobStatus{ + CompletionTime: &now, + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + }) + + Describe("#LastOperationNotSuccessful", func() { + var pred predicate.Predicate + + JustBeforeEach(func() { + pred = LastOperationNotSuccessful() + }) + + Context("when last error is not set", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + LastError: nil, + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when last error is set", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + LastError: pointer.String("foo error"), + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + }) + + Describe("#HasOperationAnnotation", func() { + var pred predicate.Predicate + + JustBeforeEach(func() { + pred = HasOperationAnnotation() + }) + + Context("when has no operation annotation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when has operation annotation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }, + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + }) + + Describe("#OR", func() { + var pred predicate.Predicate + + JustBeforeEach(func() { + pred = predicate.Or( + HasOperationAnnotation(), + LastOperationNotSuccessful(), + ) + }) + + Context("when has neither operation annotation nor last error", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when has operation annotation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }, + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + + Context("when has last error", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + Status: druidv1alpha1.EtcdStatus{ + LastError: pointer.String("error"), + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + + Context("when has both operation annotation and last error", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }, + }, + Status: druidv1alpha1.EtcdStatus{ + LastError: pointer.String("error"), + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + }) + + Describe("#EtcdReconciliationFinished", func() { + var pred predicate.Predicate + + BeforeEach(func() { + pred = EtcdReconciliationFinished(false) + }) + + Context("when etcd has no reconcile operation annotation and observedGeneration is not present", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + Generation: 2, + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when etcd has no reconcile operation annotation and observedGeneration is not equal to generation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + Generation: 2, + }, + Status: druidv1alpha1.EtcdStatus{ + ObservedGeneration: pointer.Int64(1), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when etcd has no reconcile operation annotation and observedGeneration is equal to generation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + Generation: 2, + }, + Status: druidv1alpha1.EtcdStatus{ + ObservedGeneration: pointer.Int64(2), + }, + } + }) + + It("should return true", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) + }) + }) + + Context("when etcd has reconcile operation annotation and observedGeneration is not present", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }, + Generation: 1, + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when etcd has reconcile operation annotation and observedGeneration is not equal to generation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }, + Generation: 2, + }, + Status: druidv1alpha1.EtcdStatus{ + ObservedGeneration: pointer.Int64(1), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + + Context("when etcd has reconcile operation annotation and observedGeneration is equal to generation", func() { + BeforeEach(func() { + obj = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }, + Generation: 2, + }, + Status: druidv1alpha1.EtcdStatus{ + ObservedGeneration: pointer.Int64(2), + }, + } + }) + + It("should return false", func() { + gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) + gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) + }) + }) + }) +}) diff --git a/internal/controller/secret/config.go b/internal/controller/secret/config.go new file mode 100644 index 000000000..db458dacd --- /dev/null +++ b/internal/controller/secret/config.go @@ -0,0 +1,44 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "github.com/gardener/etcd-druid/controllers/utils" + + flag "github.com/spf13/pflag" +) + +const ( + workersFlagName = "secret-workers" + + defaultWorkers = 10 +) + +// Config defines the configuration for the Secret Controller. +type Config struct { + // Workers is the number of workers concurrently processing reconciliation requests. + Workers int +} + +// InitFromFlags initializes the config from the provided CLI flag set. +func InitFromFlags(fs *flag.FlagSet, cfg *Config) { + fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, + "Number of worker threads for the secrets controller.") +} + +// Validate validates the config. +func (cfg *Config) Validate() error { + return utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers) +} diff --git a/internal/controller/secret/reconciler.go b/internal/controller/secret/reconciler.go new file mode 100644 index 000000000..d60e7b627 --- /dev/null +++ b/internal/controller/secret/reconciler.go @@ -0,0 +1,116 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// Reconciler reconciles secrets referenced in Etcd objects. +type Reconciler struct { + client.Client + Config *Config + logger logr.Logger +} + +// NewReconciler creates a new reconciler for Secret. +func NewReconciler(mgr manager.Manager, config *Config) *Reconciler { + return &Reconciler{ + Client: mgr.GetClient(), + Config: config, + logger: log.Log.WithName("secret-controller"), + } +} + +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update;patch + +// Reconcile reconciles the secret. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + secret := &corev1.Secret{} + if err := r.Get(ctx, req.NamespacedName, secret); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + logger := r.logger.WithValues("secret", client.ObjectKeyFromObject(secret)) + + etcdList := &druidv1alpha1.EtcdList{} + if err := r.Client.List(ctx, etcdList, client.InNamespace(secret.Namespace)); err != nil { + return ctrl.Result{}, err + } + + if needed, etcd := isFinalizerNeeded(secret.Name, etcdList); needed { + logger.Info("Adding finalizer for secret since it is referenced by etcd resource", + "secretNamespace", secret.Namespace, "secretName", secret.Name, "etcdNamespace", etcd.Namespace, "etcdName", etcd.Name) + return ctrl.Result{}, addFinalizer(ctx, logger, r.Client, secret) + } + return ctrl.Result{}, removeFinalizer(ctx, logger, r.Client, secret) +} + +func isFinalizerNeeded(secretName string, etcdList *druidv1alpha1.EtcdList) (bool, *druidv1alpha1.Etcd) { + for _, etcd := range etcdList.Items { + if etcd.Spec.Etcd.ClientUrlTLS != nil && + (etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name == secretName || + etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name == secretName || + etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name == secretName) { + return true, &etcd + } + + if etcd.Spec.Etcd.PeerUrlTLS != nil && + (etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name == secretName || + etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name == secretName) { // Currently, no client certificate for peer url is used in ETCD cluster + return true, &etcd + } + + if etcd.Spec.Backup.Store != nil && + etcd.Spec.Backup.Store.SecretRef != nil && + etcd.Spec.Backup.Store.SecretRef.Name == secretName { + return true, &etcd + } + } + + return false, nil +} + +func addFinalizer(ctx context.Context, logger logr.Logger, k8sClient client.Client, secret *corev1.Secret) error { + if finalizers := sets.NewString(secret.Finalizers...); finalizers.Has(common.FinalizerName) { + return nil + } + logger.Info("Adding finalizer", "namespace", secret.Namespace, "name", secret.Name, "finalizerName", common.FinalizerName) + return client.IgnoreNotFound(controllerutils.AddFinalizers(ctx, k8sClient, secret, common.FinalizerName)) +} + +func removeFinalizer(ctx context.Context, logger logr.Logger, k8sClient client.Client, secret *corev1.Secret) error { + if finalizers := sets.NewString(secret.Finalizers...); !finalizers.Has(common.FinalizerName) { + return nil + } + logger.Info("Removing finalizer", "namespace", secret.Namespace, "name", secret.Name, "finalizerName", common.FinalizerName) + return client.IgnoreNotFound(controllerutils.RemoveFinalizers(ctx, k8sClient, secret, common.FinalizerName)) +} diff --git a/internal/controller/secret/reconciler_test.go b/internal/controller/secret/reconciler_test.go new file mode 100644 index 000000000..9a87bedfa --- /dev/null +++ b/internal/controller/secret/reconciler_test.go @@ -0,0 +1,240 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "context" + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/client/kubernetes" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/gardener/gardener/pkg/utils/test/matchers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SecretController", func() { + var ( + testSecretName = "test-secret" + testNamespace = "test-namespace" + ) + + Describe("#isFinalizerNeeded", func() { + var ( + etcdList druidv1alpha1.EtcdList + ) + + BeforeEach(func() { + etcdList = druidv1alpha1.EtcdList{} + for i := 0; i < 3; i++ { + etcdList.Items = append(etcdList.Items, druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("etcd-%d", i), + Namespace: testNamespace, + }, + }) + } + }) + + It("should return false if secret is not referred to by any Etcd object", func() { + isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) + Expect(isFinalizerNeeded).To(BeFalse()) + }) + + It("should return true if secret is referred to in an Etcd object's Spec.Etcd.ClientUrlTLS section", func() { + etcdList.Items[0].Spec.Etcd.ClientUrlTLS = &druidv1alpha1.TLSConfig{ + ServerTLSSecretRef: v1.SecretReference{ + Name: testSecretName, + Namespace: testNamespace, + }, + } + isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) + Expect(isFinalizerNeeded).To(BeTrue()) + }) + + It("should return true if secret is referred to in an Etcd object's Spec.Etcd.PeerUrlTLS section", func() { + etcdList.Items[1].Spec.Etcd.PeerUrlTLS = &druidv1alpha1.TLSConfig{ + ServerTLSSecretRef: v1.SecretReference{ + Name: testSecretName, + Namespace: testNamespace, + }, + } + isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) + Expect(isFinalizerNeeded).To(BeTrue()) + }) + + It("should return true if secret is referred to in an Etcd object's Spec.Backup.Store section", func() { + etcdList.Items[2].Spec.Backup.Store = &druidv1alpha1.StoreSpec{ + SecretRef: &v1.SecretReference{ + Name: testSecretName, + Namespace: testNamespace, + }, + } + isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) + Expect(isFinalizerNeeded).To(BeTrue()) + }) + }) + + Describe("#addFinalizer", func() { + var ( + secret *corev1.Secret + ctx context.Context + logger = logr.Discard() + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + ) + + Context("test #addFinalizer on existing secret", func() { + BeforeEach(func() { + ctx = context.Background() + secret = ensureSecretCreation(ctx, testSecretName, testNamespace, fakeClient) + }) + + AfterEach(func() { + ensureSecretRemoval(ctx, testSecretName, testNamespace, fakeClient) + }) + + It("should return nil if secret already has finalizer", func() { + Expect(controllerutils.AddFinalizers(ctx, fakeClient, secret, common.FinalizerName)).To(Succeed()) + Expect(addFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) + }) + + It("should add finalizer on secret if finalizer does not exist", func() { + Expect(addFinalizer(ctx, logger, fakeClient, secret)).To(Succeed()) + finalizers := sets.NewString(secret.Finalizers...) + Expect(finalizers.Has(common.FinalizerName)).To(BeTrue()) + }) + }) + + Context("test #addFinalizer when secret does not exist", func() { + BeforeEach(func() { + ctx = context.Background() + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "non-existent", + Namespace: testNamespace, + // resource version is required as OptimisticLock is used for the merge patch + // operation used by controllerutils.AddFinalizers() + ResourceVersion: "42", + }, + } + }) + + It("should not return an error if the secret is not found", func() { + Expect(addFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) + }) + }) + }) + + Describe("#removeFinalizer", func() { + var ( + secret *corev1.Secret + ctx context.Context + logger = logr.Discard() + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + ) + + Context("test remove finalizer on an existing secret", func() { + BeforeEach(func() { + ctx = context.Background() + secret = ensureSecretCreation(ctx, testSecretName, testNamespace, fakeClient) + }) + + AfterEach(func() { + ensureSecretRemoval(ctx, testSecretName, testNamespace, fakeClient) + }) + + It("should return nil if secret does not have finalizer", func() { + Expect(controllerutils.RemoveAllFinalizers(ctx, fakeClient, secret)).Should(Succeed()) + Expect(removeFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) + }) + + It("should remove finalizer from secret if finalizer exists", func() { + Expect(removeFinalizer(ctx, logger, fakeClient, secret)).To(Succeed()) + finalizers := sets.NewString(secret.Finalizers...) + Expect(finalizers.Has(common.FinalizerName)).To(BeFalse()) + }) + }) + + Context("test remove finalizer on a non-existing secret", func() { + BeforeEach(func() { + ctx = context.Background() + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "non-existent", + Namespace: testNamespace, + }, + } + }) + + It("should not return an error if the secret is not found", func() { + Expect(removeFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) + }) + }) + + }) +}) + +func ensureSecretCreation(ctx context.Context, name, namespace string, fakeClient client.WithWatch) *corev1.Secret { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Finalizers: []string{ + common.FinalizerName, + }, + }, + } + + By("Create Secret") + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + By("Ensure Secret is created") + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(secret), secret) + }).Should(Succeed()) + + return secret +} + +func ensureSecretRemoval(ctx context.Context, name, namespace string, fakeClient client.WithWatch) { + secret := &corev1.Secret{} + if err := fakeClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret); err != nil { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + return + } + + By("Remove any existing finalizers on Secret") + Expect(controllerutils.RemoveAllFinalizers(ctx, fakeClient, secret)).To(Succeed()) + + By("Delete Secret") + Expect(fakeClient.Delete(ctx, secret)).To(Succeed()) + + By("Ensure Secret is deleted") + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(secret), secret) + }).Should(BeNotFoundError()) +} diff --git a/internal/controller/secret/register.go b/internal/controller/secret/register.go new file mode 100644 index 000000000..3c5fa8614 --- /dev/null +++ b/internal/controller/secret/register.go @@ -0,0 +1,50 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druidmapper "github.com/gardener/etcd-druid/pkg/mapper" + + "github.com/gardener/gardener/pkg/controllerutils/mapper" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const controllerName = "secret-controller" + +// RegisterWithManager registers the Secret Controller with the given controller manager. +func (r *Reconciler) RegisterWithManager(ctx context.Context, mgr ctrl.Manager) error { + c, err := ctrl. + NewControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.Config.Workers, + }). + For(&corev1.Secret{}). + Build(r) + if err != nil { + return err + } + + return c.Watch( + &source.Kind{Type: &druidv1alpha1.Etcd{}}, + mapper.EnqueueRequestsFrom(ctx, mgr.GetCache(), druidmapper.EtcdToSecret(), mapper.UpdateWithOldAndNew, c.GetLogger()), + ) +} diff --git a/internal/controller/secret/secret_suite_test.go b/internal/controller/secret/secret_suite_test.go new file mode 100644 index 000000000..a3598ae8f --- /dev/null +++ b/internal/controller/secret/secret_suite_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSecretController(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs( + t, + "Secret Controller Suite", + ) +} diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go new file mode 100644 index 000000000..fc2292774 --- /dev/null +++ b/internal/controller/utils/etcdstatus.go @@ -0,0 +1,91 @@ +package utils + +import ( + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// LastOperationErrorRecorder records etcd.Status.LastOperation and etcd.Status.LastErrors +type LastOperationErrorRecorder interface { + RecordStart(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error + RecordSuccess(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error + RecordError(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error +} + +func NewLastOperationErrorRecorder(client client.Client, logger logr.Logger) LastOperationErrorRecorder { + return &lastOpErrRecorder{ + client: client, + logger: logger, + } +} + +type lastOpErrRecorder struct { + client client.Client + objectKey client.ObjectKey + logger logr.Logger +} + +func (l *lastOpErrRecorder) RecordStart(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error { + const ( + etcdReconcileStarted string = "Etcd cluster reconciliation is in progress" + etcdDeletionStarted string = "Etcd cluster deletion is in progress" + ) + var description string + switch operationType { + case druidv1alpha1.LastOperationTypeCreate, druidv1alpha1.LastOperationTypeReconcile: + description = etcdReconcileStarted + case druidv1alpha1.LastOperationTypeDelete: + description = etcdDeletionStarted + } + return l.recordLastOperationAndErrors(ctx, etcd, operationType, druidv1alpha1.LastOperationStateProcessing, description) +} + +func (l *lastOpErrRecorder) RecordSuccess(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error { + const ( + etcdReconciledSuccessfully string = "Etcd cluster has been successfully reconciled" + etcdDeletedSuccessfully string = "Etcd cluster has been successfully deleted" + ) + var description string + switch operationType { + case druidv1alpha1.LastOperationTypeCreate, druidv1alpha1.LastOperationTypeReconcile: + description = etcdReconciledSuccessfully + case druidv1alpha1.LastOperationTypeDelete: + description = etcdDeletedSuccessfully + } + + return l.recordLastOperationAndErrors(ctx, etcd, operationType, druidv1alpha1.LastOperationStateSucceeded, description) +} + +func (l *lastOpErrRecorder) RecordError(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { + description += " Operation will be retried." + lastErrors := druiderr.MapToLastErrors(errs) + return l.recordLastOperationAndErrors(ctx, etcd, operationType, druidv1alpha1.LastOperationStateError, description, lastErrors...) +} + +func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, operationState druidv1alpha1.LastOperationState, description string, lastErrors ...druidv1alpha1.LastError) error { + etcdPatch := client.StrategicMergeFrom(etcd.DeepCopy()) + + // update last operation + if etcd.Status.LastOperation == nil { + etcd.Status.LastOperation = &druidv1alpha1.LastOperation{} + } + etcd.Status.LastOperation.RunID = ctx.RunID + etcd.Status.LastOperation.Type = operationType + etcd.Status.LastOperation.State = operationState + etcd.Status.LastOperation.LastUpdateTime = metav1.NewTime(time.Now().UTC()) + etcd.Status.LastOperation.Description = description + // update last errors + etcd.Status.LastErrors = lastErrors + + err := l.client.Status().Patch(ctx, etcd, etcdPatch) + if err != nil { + l.logger.Error(err, "failed to update LastOperation and LastErrors") + } + return err +} diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go new file mode 100644 index 000000000..1189c02ce --- /dev/null +++ b/internal/controller/utils/reconciler.go @@ -0,0 +1,116 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "errors" + "fmt" + "path/filepath" + "time" + + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // defaultImageVector is a constant for the path to the default image vector file. + defaultImageVector = "images.yaml" +) + +// getImageYAMLPath returns the path to the image vector YAML file. +// The path to the default image vector YAML path is returned, unless `useEtcdWrapperImageVector` +// is set to true, in which case the path to the etcd wrapper image vector YAML is returned. +func getImageYAMLPath() string { + return filepath.Join(common.ChartPath, defaultImageVector) +} + +// CreateImageVector creates an image vector from the default images.yaml file or the images-wrapper.yaml file. +func CreateImageVector() (imagevector.ImageVector, error) { + imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(getImageYAMLPath()) + if err != nil { + return nil, err + } + return imageVector, nil +} + +// ContainsFinalizer checks if an object has a finalizer present on it. +// TODO: With the controller-runtime version 0.16.x onwards this is provided by controllerutil.ContainsFinalizer. +// TODO: Remove this function once we move to this version. +func ContainsFinalizer(o client.Object, finalizer string) bool { + finalizers := o.GetFinalizers() + for _, f := range finalizers { + if f == finalizer { + return true + } + } + return false +} + +type ReconcileStepResult struct { + result ctrl.Result + errs []error + description string + continueReconcile bool +} + +func (r ReconcileStepResult) ReconcileResult() (ctrl.Result, error) { + return r.result, errors.Join(r.errs...) +} + +func (r ReconcileStepResult) GetErrors() []error { + return r.errs +} + +func (r ReconcileStepResult) GetDescription() string { + if len(r.errs) > 0 { + return fmt.Sprintf("%s %s", r.description, errors.Join(r.errs...).Error()) + } + return r.description +} + +func DoNotRequeue() ReconcileStepResult { + return ReconcileStepResult{ + continueReconcile: false, + result: ctrl.Result{Requeue: false}, + } +} + +func ContinueReconcile() ReconcileStepResult { + return ReconcileStepResult{ + continueReconcile: true, + } +} + +func ReconcileWithError(errs ...error) ReconcileStepResult { + return ReconcileStepResult{ + continueReconcile: false, + result: ctrl.Result{Requeue: true}, + errs: errs, + } +} + +func ReconcileAfter(period time.Duration, description string) ReconcileStepResult { + return ReconcileStepResult{ + continueReconcile: false, + result: ctrl.Result{RequeueAfter: period}, + description: description, + } +} + +func ShortCircuitReconcileFlow(result ReconcileStepResult) bool { + return !result.continueReconcile +} diff --git a/internal/controller/utils/utils_suite_test.go b/internal/controller/utils/utils_suite_test.go new file mode 100644 index 000000000..f825cd8e7 --- /dev/null +++ b/internal/controller/utils/utils_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestControllerUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controller Utils suite") +} diff --git a/internal/controller/utils/validator.go b/internal/controller/utils/validator.go new file mode 100644 index 000000000..d3ec4d324 --- /dev/null +++ b/internal/controller/utils/validator.go @@ -0,0 +1,50 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + + "golang.org/x/exp/constraints" +) + +// ShouldBeOneOfAllowedValues checks if value is amongst the allowedValues. If it is not then an error is returned else nil is returned. +// Type is constrained by comparable forcing the consumers to only use concrete types that can be compared using the == or != operators. +func ShouldBeOneOfAllowedValues[E comparable](key string, allowedValues []E, value E) error { + for _, av := range allowedValues { + if av == value { + return nil + } + } + return fmt.Errorf("unsupported value %v provided for %s. allowed values are: %v", value, key, allowedValues) +} + +// MustBeGreaterThan checks if the value is greater than the lowerBound. If it is not then an error is returned else nil is returned. +// Type is constrained by constraints.Ordered which enforces the consumers to use concrete types that can be compared using >, >=, <, <= operators. +func MustBeGreaterThan[E constraints.Ordered](key string, lowerBound, value E) error { + if value <= lowerBound { + return fmt.Errorf("%s should have a value greater than %v. value provided is %v", key, lowerBound, value) + } + return nil +} + +// MustBeGreaterThanOrEqualTo checks if the value is greater than or equal to the lowerBound. If it is not then an error is returned else nil is returned. +// Type is constrained by constraints.Ordered which enforces the consumers to use concrete types that can be compared using >, >=, <, <= operators. +func MustBeGreaterThanOrEqualTo[E constraints.Ordered](key string, lowerBound, value E) error { + if value < lowerBound { + return fmt.Errorf("%s should have a value greater or equal to %v. value provided is %v", key, lowerBound, value) + } + return nil +} diff --git a/internal/controller/utils/validator_test.go b/internal/controller/utils/validator_test.go new file mode 100644 index 000000000..db1f63d30 --- /dev/null +++ b/internal/controller/utils/validator_test.go @@ -0,0 +1,53 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Validator Tests", func() { + + DescribeTable("#ShouldBeOneOfAllowedValues tests", + func(key string, allowedValues []string, valueToCheck string, expectError bool) { + err := ShouldBeOneOfAllowedValues(key, allowedValues, valueToCheck) + Expect(err != nil).To(Equal(expectError)) + }, + Entry("value is not present in the allowed values slice", "locknames", []string{"configmaps"}, "leases", true), + Entry("values is present in the allowed values slice", "locknames", []string{"configmaps", "leases", "endpoints"}, "leases", false), + ) + + DescribeTable("#MustBeGreaterThan", + func(key string, lowerBound int, value int, expectError bool) { + err := MustBeGreaterThan(key, lowerBound, value) + Expect(err != nil).To(Equal(expectError)) + }, + Entry("value is not greater than the lower bound", "workers", 0, -1, true), + Entry("value is greater than the lower bound", "durationSeconds", 10, 20, false), + Entry("value is equal to the lower bound", "threshold", 10, 10, true), + ) + + DescribeTable("#MustBeGreaterThanOrEqualTo", + func(key string, lowerBound int, value int, expectError bool) { + err := MustBeGreaterThanOrEqualTo(key, lowerBound, value) + Expect(err != nil).To(Equal(expectError)) + }, + Entry("value is not greater than the lower bound", "workers", 0, -1, true), + Entry("value is greater than the lower bound", "durationSeconds", 10, 20, false), + Entry("value is equal to the lower bound", "threshold", 10, 10, false), + ) + +}) diff --git a/internal/errors/errors.go b/internal/errors/errors.go new file mode 100644 index 000000000..d951b9ec1 --- /dev/null +++ b/internal/errors/errors.go @@ -0,0 +1,49 @@ +package errors + +import ( + "errors" + "fmt" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type DruidError struct { + Code druidv1alpha1.ErrorCode + Cause error + Operation string + Message string +} + +func (r *DruidError) Error() string { + return fmt.Sprintf("[Operation: %s, Code: %s] %s", r.Operation, r.Code, r.Cause.Error()) +} + +func WrapError(err error, code druidv1alpha1.ErrorCode, operation string, message string) error { + if err == nil { + return nil + } + return &DruidError{ + Code: code, + Cause: err, + Operation: operation, + Message: message, + } +} + +func MapToLastErrors(errs []error) []druidv1alpha1.LastError { + lastErrs := make([]druidv1alpha1.LastError, 0, len(errs)) + for _, err := range errs { + druidErr := &DruidError{} + if errors.As(err, &druidErr) { + lastErr := druidv1alpha1.LastError{ + Code: druidErr.Code, + Description: druidErr.Message, + LastUpdateTime: metav1.NewTime(time.Now().UTC()), + } + lastErrs = append(lastErrs, lastErr) + } + } + return lastErrs +} diff --git a/internal/features/features.go b/internal/features/features.go new file mode 100644 index 000000000..a7941de36 --- /dev/null +++ b/internal/features/features.go @@ -0,0 +1,44 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package features + +import ( + "k8s.io/component-base/featuregate" +) + +const ( + // Every feature should add method here following this template: + // + // // MyFeature enables Foo. + // // owner: @username + // // alpha: v0.X + // MyFeature featuregate.Feature = "MyFeature" + + // UseEtcdWrapper enables the use of etcd-wrapper image and a compatible version + // of etcd-backup-restore, along with component-specific configuration + // changes required for the usage of the etcd-wrapper image. + // owner @unmarshall @aaronfern + // alpha: v0.19 + UseEtcdWrapper featuregate.Feature = "UseEtcdWrapper" +) + +var defaultFeatures = map[featuregate.Feature]featuregate.FeatureSpec{ + UseEtcdWrapper: {Default: false, PreRelease: featuregate.Alpha}, +} + +// GetDefaultFeatures returns the default feature gates known to etcd-druid. +func GetDefaultFeatures() map[featuregate.Feature]featuregate.FeatureSpec { + return defaultFeatures +} diff --git a/internal/health/condition/builder.go b/internal/health/condition/builder.go new file mode 100644 index 000000000..a26d6cb70 --- /dev/null +++ b/internal/health/condition/builder.go @@ -0,0 +1,141 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition + +import ( + "sort" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// skipMergeConditions contain the list of conditions we dont want to add to the list if not recalculated +var skipMergeConditions = map[druidv1alpha1.ConditionType]struct{}{ + druidv1alpha1.ConditionTypeReady: {}, + druidv1alpha1.ConditionTypeAllMembersReady: {}, + druidv1alpha1.ConditionTypeBackupReady: {}, +} + +// Builder is an interface for building conditions. +type Builder interface { + WithOldConditions(conditions []druidv1alpha1.Condition) Builder + WithResults(result []Result) Builder + WithNowFunc(now func() metav1.Time) Builder + Build(replicas int32) []druidv1alpha1.Condition +} + +type defaultBuilder struct { + old map[druidv1alpha1.ConditionType]druidv1alpha1.Condition + results map[druidv1alpha1.ConditionType]Result + nowFunc func() metav1.Time +} + +// NewBuilder returns a Builder for a specific condition. +func NewBuilder() Builder { + return &defaultBuilder{ + old: make(map[druidv1alpha1.ConditionType]druidv1alpha1.Condition), + results: make(map[druidv1alpha1.ConditionType]Result), + nowFunc: func() metav1.Time { + return metav1.NewTime(time.Now().UTC()) + }, + } +} + +// WithOldConditions sets the old conditions. It can be used to provide default values. +func (b *defaultBuilder) WithOldConditions(conditions []druidv1alpha1.Condition) Builder { + for _, cond := range conditions { + b.old[cond.Type] = cond + } + + return b +} + +// WithResults adds the results. +func (b *defaultBuilder) WithResults(results []Result) Builder { + for _, result := range results { + if result == nil { + continue + } + b.results[result.ConditionType()] = result + } + + return b +} + +// WithNowFunc sets the function used for getting the current time. +// Should only be used for tests. +func (b *defaultBuilder) WithNowFunc(now func() metav1.Time) Builder { + b.nowFunc = now + return b +} + +// Build creates the conditions. +// It merges the existing conditions with the results added to the builder. +// If OldCondition is provided: +// - Any changes to status set the `LastTransitionTime` +// - `LastUpdateTime` is always set. +func (b *defaultBuilder) Build(replicas int32) []druidv1alpha1.Condition { + var ( + now = b.nowFunc() + conditions []druidv1alpha1.Condition + ) + + for condType, res := range b.results { + condition, ok := b.old[condType] + if !ok { + condition = druidv1alpha1.Condition{ + Type: condType, + LastTransitionTime: now, + } + } + + if condition.Status != res.Status() { + condition.LastTransitionTime = now + } + condition.LastUpdateTime = now + if replicas == 0 { + if condition.Status == "" { + condition.Status = druidv1alpha1.ConditionUnknown + } + condition.Reason = ConditionNotChecked + condition.Message = "etcd cluster has been scaled down" + } else { + condition.Status = res.Status() + condition.Message = res.Message() + condition.Reason = res.Reason() + } + + conditions = append(conditions, condition) + delete(b.old, condType) + } + + for _, condition := range b.old { + // Do not add conditions that are part of the skipMergeConditions list + _, ok := skipMergeConditions[condition.Type] + if ok { + continue + } + // Add existing conditions as they were. This needs to be changed when SSA is used. + conditions = append(conditions, condition) + } + + sort.Slice(conditions, func(i, j int) bool { + return conditions[i].Type < conditions[j].Type + }) + + return conditions +} diff --git a/internal/health/condition/builder_test.go b/internal/health/condition/builder_test.go new file mode 100644 index 000000000..d9f868c5a --- /dev/null +++ b/internal/health/condition/builder_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/health/condition" +) + +var _ = Describe("Builder", func() { + var ( + builder Builder + now time.Time + ) + + BeforeEach(func() { + now, _ = time.Parse(time.RFC3339, "2021-06-01") + builder = NewBuilder() + }) + + JustBeforeEach(func() { + builder.WithNowFunc(func() metav1.Time { + return metav1.NewTime(now) + }) + }) + + Describe("#Build", func() { + Context("when Builder has old conditions", func() { + var ( + oldConditionTime time.Time + oldConditions []druidv1alpha1.Condition + ) + BeforeEach(func() { + oldConditionTime = now.Add(-12 * time.Hour) + + oldConditions = []druidv1alpha1.Condition{ + { + Type: druidv1alpha1.ConditionTypeAllMembersReady, + LastUpdateTime: metav1.NewTime(oldConditionTime), + LastTransitionTime: metav1.NewTime(oldConditionTime), + Status: druidv1alpha1.ConditionTrue, + Reason: "foo reason", + Message: "foo message", + }, + { + Type: druidv1alpha1.ConditionTypeReady, + LastUpdateTime: metav1.NewTime(oldConditionTime), + LastTransitionTime: metav1.NewTime(oldConditionTime), + Status: druidv1alpha1.ConditionFalse, + Reason: "bar reason", + Message: "bar message", + }, + { + Type: druidv1alpha1.ConditionTypeBackupReady, + LastUpdateTime: metav1.NewTime(oldConditionTime), + LastTransitionTime: metav1.NewTime(oldConditionTime), + Status: druidv1alpha1.ConditionTrue, + Reason: "foobar reason", + Message: "foobar message", + }, + } + + builder.WithOldConditions(oldConditions) + }) + + It("should not add old conditions", func() { + builder.WithResults([]Result{ + &result{ + ConType: druidv1alpha1.ConditionTypeAllMembersReady, + ConStatus: druidv1alpha1.ConditionTrue, + ConReason: "new reason", + ConMessage: "new message", + }, + &result{ + ConType: druidv1alpha1.ConditionTypeReady, + ConStatus: druidv1alpha1.ConditionTrue, + ConReason: "new reason", + ConMessage: "new message", + }, + }) + + conditions := builder.Build(1) + + Expect(conditions).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeAllMembersReady), + "LastUpdateTime": Equal(metav1.NewTime(now)), + "LastTransitionTime": Equal(metav1.NewTime(oldConditionTime)), + "Status": Equal(druidv1alpha1.ConditionTrue), + "Reason": Equal("new reason"), + "Message": Equal("new message"), + }), + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeReady), + "LastUpdateTime": Equal(metav1.NewTime(now)), + "LastTransitionTime": Equal(metav1.NewTime(now)), + "Status": Equal(druidv1alpha1.ConditionTrue), + "Reason": Equal("new reason"), + "Message": Equal("new message"), + }), + )) + }) + }) + + Context("when Builder has no old conditions", func() { + It("should correctly set the new conditions", func() { + builder.WithResults([]Result{ + &result{ + ConType: druidv1alpha1.ConditionTypeAllMembersReady, + ConStatus: druidv1alpha1.ConditionTrue, + ConReason: "new reason", + ConMessage: "new message", + }, + &result{ + ConType: druidv1alpha1.ConditionTypeReady, + ConStatus: druidv1alpha1.ConditionTrue, + ConReason: "new reason", + ConMessage: "new message", + }, + }) + + conditions := builder.Build(1) + + Expect(conditions).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeAllMembersReady), + "LastUpdateTime": Equal(metav1.NewTime(now)), + "LastTransitionTime": Equal(metav1.NewTime(now)), + "Status": Equal(druidv1alpha1.ConditionTrue), + "Reason": Equal("new reason"), + "Message": Equal("new message"), + }), + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeReady), + "LastUpdateTime": Equal(metav1.NewTime(now)), + "LastTransitionTime": Equal(metav1.NewTime(now)), + "Status": Equal(druidv1alpha1.ConditionTrue), + "Reason": Equal("new reason"), + "Message": Equal("new message"), + }), + )) + }) + }) + }) +}) + +type result struct { + ConType druidv1alpha1.ConditionType + ConStatus druidv1alpha1.ConditionStatus + ConReason string + ConMessage string +} + +func (r *result) ConditionType() druidv1alpha1.ConditionType { + return r.ConType +} + +func (r *result) Status() druidv1alpha1.ConditionStatus { + return r.ConStatus +} + +func (r *result) Reason() string { + return r.ConReason +} + +func (r *result) Message() string { + return r.ConMessage +} diff --git a/internal/health/condition/check_all_members.go b/internal/health/condition/check_all_members.go new file mode 100644 index 000000000..b6f270cbb --- /dev/null +++ b/internal/health/condition/check_all_members.go @@ -0,0 +1,69 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type allMembersReady struct{} + +func (a *allMembersReady) Check(_ context.Context, etcd druidv1alpha1.Etcd) Result { + if len(etcd.Status.Members) == 0 { + return &result{ + conType: druidv1alpha1.ConditionTypeAllMembersReady, + status: druidv1alpha1.ConditionUnknown, + reason: "NoMembersInStatus", + message: "Cannot determine readiness since status has no members", + } + } + + result := &result{ + conType: druidv1alpha1.ConditionTypeAllMembersReady, + status: druidv1alpha1.ConditionFalse, + reason: "NotAllMembersReady", + message: "At least one member is not ready", + } + + if int32(len(etcd.Status.Members)) < etcd.Spec.Replicas { + // not all members are registered yet + return result + } + + // If we are here this means that all members have registered. Check if any member + // has a not-ready status. If there is at least one then set the overall status as false. + ready := true + for _, member := range etcd.Status.Members { + ready = ready && member.Status == druidv1alpha1.EtcdMemberStatusReady + if !ready { + break + } + } + if ready { + result.status = druidv1alpha1.ConditionTrue + result.reason = "AllMembersReady" + result.message = "All members are ready" + } + + return result +} + +// AllMembersCheck returns a check for the "AllMembersReady" condition. +func AllMembersCheck(_ client.Client) Checker { + return &allMembersReady{} +} diff --git a/internal/health/condition/check_all_members_test.go b/internal/health/condition/check_all_members_test.go new file mode 100644 index 000000000..d62967558 --- /dev/null +++ b/internal/health/condition/check_all_members_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/health/condition" +) + +var _ = Describe("AllMembersReadyCheck", func() { + Describe("#Check", func() { + var readyMember, notReadyMember druidv1alpha1.EtcdMemberStatus + + BeforeEach(func() { + readyMember = druidv1alpha1.EtcdMemberStatus{ + Status: druidv1alpha1.EtcdMemberStatusReady, + } + notReadyMember = druidv1alpha1.EtcdMemberStatus{ + Status: druidv1alpha1.EtcdMemberStatusNotReady, + } + }) + + Context("when members in status", func() { + It("should return that all members are ready", func() { + etcd := druidv1alpha1.Etcd{ + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 3, + }, + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + readyMember, + readyMember, + }, + }, + } + check := AllMembersCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + }) + + It("should return that members are not ready", func() { + etcd := druidv1alpha1.Etcd{ + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 3, + }, + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + notReadyMember, + readyMember, + }, + }, + } + check := AllMembersCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) + }) + + It("should return all members are not ready when number of members registered are less than spec replicas", func() { + etcd := druidv1alpha1.Etcd{ + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 3, + }, + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + readyMember, + }, + }, + } + check := AllMembersCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) + Expect(result.Reason()).To(Equal("NotAllMembersReady")) + }) + }) + + Context("when no members in status", func() { + It("should return that readiness is unknown", func() { + etcd := druidv1alpha1.Etcd{ + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 3, + }, + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{}, + }, + } + check := AllMembersCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + }) + }) + }) +}) diff --git a/internal/health/condition/check_backup_ready.go b/internal/health/condition/check_backup_ready.go new file mode 100644 index 000000000..077adaea6 --- /dev/null +++ b/internal/health/condition/check_backup_ready.go @@ -0,0 +1,142 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition + +import ( + "context" + "fmt" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + coordinationv1 "k8s.io/api/coordination/v1" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type backupReadyCheck struct { + cl client.Client +} + +const ( + // BackupSucceeded is a constant that means that etcd backup has been successfully taken + BackupSucceeded string = "BackupSucceeded" + // BackupFailed is a constant that means that etcd backup has failed + BackupFailed string = "BackupFailed" + // Unknown is a constant that means that the etcd backup status is currently not known + Unknown string = "Unknown" + // ConditionNotChecked is a constant that means that the etcd backup status has not been updated or rechecked + ConditionNotChecked string = "ConditionNotChecked" +) + +func (a *backupReadyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) Result { + //Default case + result := &result{ + conType: druidv1alpha1.ConditionTypeBackupReady, + status: druidv1alpha1.ConditionUnknown, + reason: Unknown, + message: "Cannot determine etcd backup status", + } + + // Special case of etcd not being configured to take snapshots + // Do not add the BackupReady condition if backup is not configured + if etcd.Spec.Backup.Store == nil || etcd.Spec.Backup.Store.Provider == nil || len(*etcd.Spec.Backup.Store.Provider) == 0 { + return nil + } + + //Fetch snapshot leases + var ( + fullSnapErr, incrSnapErr error + fullSnapLease = &coordinationv1.Lease{} + deltaSnapLease = &coordinationv1.Lease{} + ) + fullSnapErr = a.cl.Get(ctx, types.NamespacedName{Name: getFullSnapLeaseName(&etcd), Namespace: etcd.ObjectMeta.Namespace}, fullSnapLease) + incrSnapErr = a.cl.Get(ctx, types.NamespacedName{Name: getDeltaSnapLeaseName(&etcd), Namespace: etcd.ObjectMeta.Namespace}, deltaSnapLease) + + //Set status to Unknown if errors in fetching snapshot leases or lease never renewed + if fullSnapErr != nil || incrSnapErr != nil || (fullSnapLease.Spec.RenewTime == nil && deltaSnapLease.Spec.RenewTime == nil) { + return result + } + + deltaLeaseRenewTime := deltaSnapLease.Spec.RenewTime + fullLeaseRenewTime := fullSnapLease.Spec.RenewTime + fullLeaseCreateTime := &fullSnapLease.ObjectMeta.CreationTimestamp + + if fullLeaseRenewTime == nil && deltaLeaseRenewTime != nil { + // Most probable during reconcile of existing clusters if fresh leases are created + // Treat backup as succeeded if delta snap lease renewal happens in the required time window and full snap lease is not older than 24h. + if time.Since(deltaLeaseRenewTime.Time) < 2*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration && time.Since(fullLeaseCreateTime.Time) < 24*time.Hour { + result.reason = BackupSucceeded + result.message = "Delta snapshot backup succeeded" + result.status = druidv1alpha1.ConditionTrue + return result + } + } else if deltaLeaseRenewTime == nil && fullLeaseRenewTime != nil { + //Most probable during a startup scenario for new clusters + //Special case. Return Unknown condition for some time to allow delta backups to start up + if time.Since(fullLeaseRenewTime.Time) > 5*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration { + result.message = "Periodic delta snapshots not started yet" + return result + } + } else if deltaLeaseRenewTime != nil && fullLeaseRenewTime != nil { + //Both snap leases are maintained. Both are expected to be renewed periodically + if time.Since(deltaLeaseRenewTime.Time) < 2*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration && time.Since(fullLeaseRenewTime.Time) < 24*time.Hour { + result.reason = BackupSucceeded + result.message = "Snapshot backup succeeded" + result.status = druidv1alpha1.ConditionTrue + return result + } + } + + //Cases where snapshot leases are not updated for a long time + //If snapshot leases are present and leases aren't updated, it is safe to assume that backup is not healthy + + if etcd.Status.Conditions != nil { + var prevBackupReadyStatus druidv1alpha1.Condition + for _, prevBackupReadyStatus = range etcd.Status.Conditions { + if prevBackupReadyStatus.Type == druidv1alpha1.ConditionTypeBackupReady { + break + } + } + + // Transition to "False" state only if present state is "Unknown" or "False" + if deltaLeaseRenewTime != nil && (prevBackupReadyStatus.Status == druidv1alpha1.ConditionUnknown || prevBackupReadyStatus.Status == druidv1alpha1.ConditionFalse) { + if time.Since(deltaLeaseRenewTime.Time) > 3*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration { + result.status = druidv1alpha1.ConditionFalse + result.reason = BackupFailed + result.message = "Stale snapshot leases. Not renewed in a long time" + return result + } + } + } + + //Transition to "Unknown" state is we cannot prove a "True" state + return result +} + +func getDeltaSnapLeaseName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-delta-snap", string(etcd.ObjectMeta.Name)) +} + +func getFullSnapLeaseName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-full-snap", string(etcd.ObjectMeta.Name)) +} + +// BackupReadyCheck returns a check for the "BackupReady" condition. +func BackupReadyCheck(cl client.Client) Checker { + return &backupReadyCheck{ + cl: cl, + } +} diff --git a/internal/health/condition/check_backup_ready_test.go b/internal/health/condition/check_backup_ready_test.go new file mode 100644 index 000000000..5e55c7224 --- /dev/null +++ b/internal/health/condition/check_backup_ready_test.go @@ -0,0 +1,275 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition_test + +import ( + "context" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/health/condition" + mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +var _ = Describe("BackupReadyCheck", func() { + Describe("#Check", func() { + var ( + storageProvider druidv1alpha1.StorageProvider = "testStorageProvider" + mockCtrl *gomock.Controller + cl *mockclient.MockClient + holderIDString = "123455" + noLeaseError = apierrors.StatusError{ + ErrStatus: v1.Status{ + Reason: v1.StatusReasonNotFound, + }, + } + deltaSnapshotDuration = 2 * time.Minute + + etcd = druidv1alpha1.Etcd{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-etcd", + Namespace: "default", + }, + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 1, + Backup: druidv1alpha1.BackupSpec{ + DeltaSnapshotPeriod: &v1.Duration{ + Duration: deltaSnapshotDuration, + }, + Store: &druidv1alpha1.StoreSpec{ + Prefix: "test-prefix", + Provider: &storageProvider, + }, + }, + }, + Status: druidv1alpha1.EtcdStatus{}, + } + lease = coordinationv1.Lease{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-etcd-snap", + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &holderIDString, + RenewTime: &v1.MicroTime{ + Time: time.Now(), + }, + }, + } + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + cl = mockclient.NewMockClient(mockCtrl) + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + Context("With no snapshot leases present", func() { + It("Should return Unknown rediness", func() { + cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, er *coordinationv1.Lease, _ ...client.GetOption) error { + return &noLeaseError + }, + ).AnyTimes() + + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).ToNot(BeNil()) + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal(Unknown)) + }) + }) + + Context("With both snapshot leases present", func() { + It("Should set status to BackupSucceeded if both leases are recently renewed", func() { + cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + return nil + }, + ).AnyTimes() + + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).ToNot(BeNil()) + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + Expect(result.Reason()).To(Equal(BackupSucceeded)) + }) + + It("Should set status to BackupSucceeded if delta snap lease is recently created and empty full snap lease has been created in the last 24h", func() { + cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-full-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + le.Spec.RenewTime = nil + le.Spec.HolderIdentity = nil + le.ObjectMeta.CreationTimestamp = v1.Now() + return nil + }, + ).AnyTimes() + cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-delta-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + return nil + }, + ).AnyTimes() + + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).ToNot(BeNil()) + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + Expect(result.Reason()).To(Equal(BackupSucceeded)) + }) + + It("Should set status to Unknown if empty delta snap lease is present but full snap lease is renewed recently", func() { + cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-full-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + le.Spec.RenewTime = &v1.MicroTime{Time: lease.Spec.RenewTime.Time.Add(-5 * deltaSnapshotDuration)} + return nil + }, + ).AnyTimes() + cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-delta-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + le.Spec.RenewTime = nil + le.Spec.HolderIdentity = nil + return nil + }, + ).AnyTimes() + + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).ToNot(BeNil()) + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal(Unknown)) + Expect(result.Message()).To(Equal("Periodic delta snapshots not started yet")) + }) + + It("Should set status to Unknown if both leases are stale", func() { + cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + le.Spec.RenewTime = &v1.MicroTime{ + Time: time.Now().Add(-10 * time.Minute), + } + return nil + }, + ).AnyTimes() + + etcd.Status.Conditions = []druidv1alpha1.Condition{ + { + Type: druidv1alpha1.ConditionTypeBackupReady, + Status: druidv1alpha1.ConditionTrue, + Message: "True", + }, + } + + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).ToNot(BeNil()) + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal(Unknown)) + }) + + It("Should set status to BackupFailed if both leases are stale and current condition is Unknown", func() { + cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { + *le = lease + le.Spec.RenewTime = &v1.MicroTime{ + Time: time.Now().Add(-10 * time.Minute), + } + return nil + }, + ).AnyTimes() + + etcd.Status.Conditions = []druidv1alpha1.Condition{ + { + Type: druidv1alpha1.ConditionTypeBackupReady, + Status: druidv1alpha1.ConditionUnknown, + Message: "Unknown", + }, + } + + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).ToNot(BeNil()) + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) + Expect(result.Reason()).To(Equal(BackupFailed)) + }) + }) + + Context("With no backup store configured", func() { + It("Should return nil condition", func() { + cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, er *coordinationv1.Lease) error { + return &noLeaseError + }, + ).AnyTimes() + + etcd.Spec.Backup.Store = nil + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).To(BeNil()) + etcd.Spec.Backup.Store = &druidv1alpha1.StoreSpec{ + Prefix: "test-prefix", + Provider: &storageProvider, + } + }) + }) + + Context("With backup store is configured but provider is nil", func() { + It("Should return nil condition", func() { + cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, er *coordinationv1.Lease) error { + return &noLeaseError + }, + ).AnyTimes() + + etcd.Spec.Backup.Store.Provider = nil + check := BackupReadyCheck(cl) + result := check.Check(context.TODO(), etcd) + + Expect(result).To(BeNil()) + etcd.Spec.Backup.Store.Provider = &storageProvider + }) + }) + }) +}) diff --git a/internal/health/condition/check_ready.go b/internal/health/condition/check_ready.go new file mode 100644 index 000000000..e0ef3d1b2 --- /dev/null +++ b/internal/health/condition/check_ready.go @@ -0,0 +1,71 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type readyCheck struct{} + +func (r *readyCheck) Check(_ context.Context, etcd druidv1alpha1.Etcd) Result { + + // TODO: remove this case as soon as leases are completely supported by etcd-backup-restore + if len(etcd.Status.Members) == 0 { + return &result{ + conType: druidv1alpha1.ConditionTypeReady, + status: druidv1alpha1.ConditionUnknown, + reason: "NoMembersInStatus", + message: "Cannot determine readiness since status has no members", + } + } + + var ( + size = len(etcd.Status.Members) + quorum = size/2 + 1 + readyMembers = 0 + ) + + for _, member := range etcd.Status.Members { + if member.Status == druidv1alpha1.EtcdMemberStatusNotReady { + continue + } + readyMembers++ + } + + if readyMembers < quorum { + return &result{ + conType: druidv1alpha1.ConditionTypeReady, + status: druidv1alpha1.ConditionFalse, + reason: "QuorumLost", + message: "The majority of ETCD members is not ready", + } + } + + return &result{ + conType: druidv1alpha1.ConditionTypeReady, + status: druidv1alpha1.ConditionTrue, + reason: "Quorate", + message: "The majority of ETCD members is ready", + } +} + +// ReadyCheck returns a check for the "Ready" condition. +func ReadyCheck(_ client.Client) Checker { + return &readyCheck{} +} diff --git a/internal/health/condition/check_ready_test.go b/internal/health/condition/check_ready_test.go new file mode 100644 index 000000000..03fb65d32 --- /dev/null +++ b/internal/health/condition/check_ready_test.go @@ -0,0 +1,135 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition_test + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/health/condition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ReadyCheck", func() { + Describe("#Check", func() { + var readyMember, notReadyMember, unknownMember druidv1alpha1.EtcdMemberStatus + + BeforeEach(func() { + readyMember = druidv1alpha1.EtcdMemberStatus{ + Status: druidv1alpha1.EtcdMemberStatusReady, + } + notReadyMember = druidv1alpha1.EtcdMemberStatus{ + Status: druidv1alpha1.EtcdMemberStatusNotReady, + } + unknownMember = druidv1alpha1.EtcdMemberStatus{ + Status: druidv1alpha1.EtcdMemberStatusUnknown, + } + }) + + Context("when members in status", func() { + It("should return that the cluster has a quorum (all members ready)", func() { + etcd := druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + readyMember, + readyMember, + }, + }, + } + check := ReadyCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + }) + + It("should return that the cluster has a quorum (members are partly unknown)", func() { + etcd := druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + unknownMember, + unknownMember, + }, + }, + } + check := ReadyCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + }) + + It("should return that the cluster has a quorum (one member not ready)", func() { + etcd := druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + notReadyMember, + readyMember, + }, + }, + } + check := ReadyCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + }) + + It("should return that the cluster has lost its quorum", func() { + etcd := druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{ + readyMember, + notReadyMember, + notReadyMember, + }, + }, + } + check := ReadyCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) + Expect(result.Reason()).To(Equal("QuorumLost")) + }) + }) + + Context("when no members in status", func() { + It("should return that quorum is unknown", func() { + etcd := druidv1alpha1.Etcd{ + Status: druidv1alpha1.EtcdStatus{ + Members: []druidv1alpha1.EtcdMemberStatus{}, + }, + } + check := ReadyCheck(nil) + + result := check.Check(context.TODO(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal("NoMembersInStatus")) + }) + }) + }) + +}) diff --git a/internal/health/condition/condition_suite_test.go b/internal/health/condition/condition_suite_test.go new file mode 100644 index 000000000..ff4383fdb --- /dev/null +++ b/internal/health/condition/condition_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCondition(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Condition Suite") +} diff --git a/internal/health/condition/types.go b/internal/health/condition/types.go new file mode 100644 index 000000000..98c3a3b7a --- /dev/null +++ b/internal/health/condition/types.go @@ -0,0 +1,57 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +) + +// Checker is an interface to check the etcd resource and to return condition results. +type Checker interface { + Check(ctx context.Context, etcd druidv1alpha1.Etcd) Result +} + +// Result encapsulates a condition result +type Result interface { + ConditionType() druidv1alpha1.ConditionType + Status() druidv1alpha1.ConditionStatus + Reason() string + Message() string +} + +type result struct { + conType druidv1alpha1.ConditionType + status druidv1alpha1.ConditionStatus + reason string + message string +} + +func (r *result) ConditionType() druidv1alpha1.ConditionType { + return r.conType +} + +func (r *result) Status() druidv1alpha1.ConditionStatus { + return r.status +} + +func (r *result) Reason() string { + return r.reason +} + +func (r *result) Message() string { + return r.message +} diff --git a/internal/health/etcdmember/builder.go b/internal/health/etcdmember/builder.go new file mode 100644 index 000000000..fa9a77052 --- /dev/null +++ b/internal/health/etcdmember/builder.go @@ -0,0 +1,115 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdmember + +import ( + "sort" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Builder is an interface for building status objects for etcd members. +type Builder interface { + WithOldMembers(members []druidv1alpha1.EtcdMemberStatus) Builder + WithResults(results []Result) Builder + WithNowFunc(now func() metav1.Time) Builder + Build() []druidv1alpha1.EtcdMemberStatus +} + +type defaultBuilder struct { + old map[string]druidv1alpha1.EtcdMemberStatus + results map[string]Result + nowFunc func() metav1.Time +} + +// NewBuilder returns a Builder for a specific etcd member status. +func NewBuilder() Builder { + return &defaultBuilder{ + old: make(map[string]druidv1alpha1.EtcdMemberStatus), + results: make(map[string]Result), + nowFunc: func() metav1.Time { + return metav1.NewTime(time.Now().UTC()) + }, + } +} + +// WithOldMember sets the old etcd member statuses. It can be used to provide default values. +func (b *defaultBuilder) WithOldMembers(members []druidv1alpha1.EtcdMemberStatus) Builder { + for _, member := range members { + b.old[member.Name] = member + } + + return b +} + +// WithResults adds the results. +func (b *defaultBuilder) WithResults(results []Result) Builder { + for _, res := range results { + if res == nil { + continue + } + b.results[res.Name()] = res + } + + return b +} + +// WithNowFunc sets the function used for getting the current time. +// Should only be used for tests. +func (b *defaultBuilder) WithNowFunc(now func() metav1.Time) Builder { + b.nowFunc = now + return b +} + +// Build creates the etcd member statuses. +// It merges the existing members with the results added to the builder. +// If OldCondition is provided: +// - Any changes to status set the `LastTransitionTime` +func (b *defaultBuilder) Build() []druidv1alpha1.EtcdMemberStatus { + var ( + now = b.nowFunc() + + members []druidv1alpha1.EtcdMemberStatus + ) + + for name, res := range b.results { + memberStatus := druidv1alpha1.EtcdMemberStatus{ + ID: res.ID(), + Name: res.Name(), + Role: res.Role(), + Status: res.Status(), + Reason: res.Reason(), + LastTransitionTime: now, + } + + // Don't reset LastTransitionTime if status didn't change + if oldMemberStatus, ok := b.old[name]; ok { + if oldMemberStatus.Status == res.Status() { + memberStatus.LastTransitionTime = oldMemberStatus.LastTransitionTime + } + } + + members = append(members, memberStatus) + } + + sort.Slice(members, func(i, j int) bool { + return members[i].Name < members[j].Name + }) + + return members +} diff --git a/internal/health/etcdmember/builder_test.go b/internal/health/etcdmember/builder_test.go new file mode 100644 index 000000000..c0027fd52 --- /dev/null +++ b/internal/health/etcdmember/builder_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdmember_test + +import ( + "time" + + "k8s.io/utils/pointer" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/health/etcdmember" +) + +var _ = Describe("Builder", func() { + var ( + builder Builder + now time.Time + ) + + BeforeEach(func() { + now, _ = time.Parse(time.RFC3339, "2021-06-01") + builder = NewBuilder() + }) + + JustBeforeEach(func() { + builder.WithNowFunc(func() metav1.Time { + return metav1.NewTime(now) + }) + }) + + Describe("#Build", func() { + Context("when Builder has old members", func() { + var ( + oldMembers map[string]druidv1alpha1.EtcdMemberStatus + ) + BeforeEach(func() { + oldMembers = map[string]druidv1alpha1.EtcdMemberStatus{ + "1": { + Name: "member1", + ID: pointer.String("1"), + Status: druidv1alpha1.EtcdMemberStatusReady, + Reason: "foo reason", + LastTransitionTime: metav1.NewTime(now.Add(-12 * time.Hour)), + }, + "2": { + Name: "member2", + ID: pointer.String("2"), + Status: druidv1alpha1.EtcdMemberStatusReady, + Reason: "bar reason", + LastTransitionTime: metav1.NewTime(now.Add(-6 * time.Hour)), + }, + "3": { + Name: "member3", + ID: pointer.String("3"), + Status: druidv1alpha1.EtcdMemberStatusReady, + Reason: "foobar reason", + LastTransitionTime: metav1.NewTime(now.Add(-18 * time.Hour)), + }, + } + + builder.WithOldMembers([]druidv1alpha1.EtcdMemberStatus{ + oldMembers["1"], + oldMembers["2"], + oldMembers["3"], + }) + }) + + It("should correctly set the LastTransitionTime", func() { + builder.WithResults([]Result{ + &result{ + MemberID: pointer.String("3"), + MemberName: "member3", + MemberStatus: druidv1alpha1.EtcdMemberStatusUnknown, + MemberReason: "unknown reason", + }, + }) + + conditions := builder.Build() + + Expect(conditions).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("member3"), + "ID": PointTo(Equal("3")), + "Status": Equal(druidv1alpha1.EtcdMemberStatusUnknown), + "Reason": Equal("unknown reason"), + "LastTransitionTime": Equal(metav1.NewTime(now)), + }), + )) + }) + }) + + Context("when Builder has no old members", func() { + var ( + memberRoleLeader, memberRoleMember druidv1alpha1.EtcdRole + ) + + BeforeEach(func() { + memberRoleLeader = druidv1alpha1.EtcdRoleLeader + memberRoleMember = druidv1alpha1.EtcdRoleMember + }) + + It("should not add any members but sort them", func() { + builder.WithResults([]Result{ + &result{ + MemberID: pointer.String("2"), + MemberName: "member2", + MemberRole: &memberRoleMember, + MemberStatus: druidv1alpha1.EtcdMemberStatusReady, + MemberReason: "foo reason", + }, + &result{ + MemberID: pointer.String("1"), + MemberName: "member1", + MemberRole: &memberRoleLeader, + MemberStatus: druidv1alpha1.EtcdMemberStatusUnknown, + MemberReason: "unknown reason", + }, + }) + + conditions := builder.Build() + + Expect(conditions).To(HaveLen(2)) + Expect(conditions[0]).To(MatchFields(IgnoreExtras, Fields{ + "Name": Equal("member1"), + "ID": PointTo(Equal("1")), + "Role": PointTo(Equal(druidv1alpha1.EtcdRoleLeader)), + "Status": Equal(druidv1alpha1.EtcdMemberStatusUnknown), + "Reason": Equal("unknown reason"), + "LastTransitionTime": Equal(metav1.NewTime(now)), + })) + Expect(conditions[1]).To(MatchFields(IgnoreExtras, Fields{ + "Name": Equal("member2"), + "ID": PointTo(Equal("2")), + "Role": PointTo(Equal(druidv1alpha1.EtcdRoleMember)), + "Status": Equal(druidv1alpha1.EtcdMemberStatusReady), + "Reason": Equal("foo reason"), + "LastTransitionTime": Equal(metav1.NewTime(now)), + })) + }) + }) + }) +}) + +type result struct { + MemberID *string + MemberName string + MemberRole *druidv1alpha1.EtcdRole + MemberStatus druidv1alpha1.EtcdMemberConditionStatus + MemberReason string +} + +func (r *result) ID() *string { + return r.MemberID +} + +func (r *result) Name() string { + return r.MemberName +} + +func (r *result) Role() *druidv1alpha1.EtcdRole { + return r.MemberRole +} + +func (r *result) Reason() string { + return r.MemberReason +} + +func (r *result) Status() druidv1alpha1.EtcdMemberConditionStatus { + return r.MemberStatus +} diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go new file mode 100644 index 000000000..b61e6f18b --- /dev/null +++ b/internal/health/etcdmember/check_ready.go @@ -0,0 +1,157 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdmember + +import ( + "context" + "strings" + "time" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/go-logr/logr" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/pkg/utils" +) + +type readyCheck struct { + logger logr.Logger + cl client.Client + etcdMemberNotReadyThreshold time.Duration + etcdMemberUnknownThreshold time.Duration +} + +// TimeNow is the function used by this check to get the current time. +var TimeNow = time.Now + +func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Result { + var ( + results []Result + checkTime = TimeNow().UTC() + ) + + leases := &coordinationv1.LeaseList{} + if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ + common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: utils.PurposeMemberLease}); err != nil { + r.logger.Error(err, "failed to get leases for etcd member readiness check") + } + + for _, lease := range leases.Items { + var ( + id, role = separateIdFromRole(lease.Spec.HolderIdentity) + res = &result{ + id: id, + name: lease.Name, + role: role, + } + ) + + // Check if member is in bootstrapping phase + // Members are supposed to be added to the members array only if they have joined the cluster (== RenewTime is set). + // This behavior is expected by the `Ready` condition and it will become imprecise if members are added here too early. + renew := lease.Spec.RenewTime + if renew == nil { + r.logger.Info("Member hasn't acquired lease yet, still in bootstrapping phase", "name", lease.Name) + continue + } + + // Check if member state must be considered as not ready + if renew.Add(r.etcdMemberUnknownThreshold).Add(r.etcdMemberNotReadyThreshold).Before(checkTime) { + res.status = druidv1alpha1.EtcdMemberStatusNotReady + res.reason = "UnknownGracePeriodExceeded" + results = append(results, res) + continue + } + + // Check if member state must be considered as unknown + if renew.Add(r.etcdMemberUnknownThreshold).Before(checkTime) { + // If pod is not running or cannot be found then we deduce that the status is NotReady. + ready, err := r.checkContainersAreReady(ctx, lease.Namespace, lease.Name) + if (err == nil && !ready) || apierrors.IsNotFound(err) { + res.status = druidv1alpha1.EtcdMemberStatusNotReady + res.reason = "ContainersNotReady" + results = append(results, res) + continue + } + + res.status = druidv1alpha1.EtcdMemberStatusUnknown + res.reason = "LeaseExpired" + results = append(results, res) + continue + } + + res.status = druidv1alpha1.EtcdMemberStatusReady + res.reason = "LeaseSucceeded" + results = append(results, res) + } + + return results +} + +const holderIdentitySeparator = ":" + +func separateIdFromRole(holderIdentity *string) (*string, *druidv1alpha1.EtcdRole) { + if holderIdentity == nil { + return nil, nil + } + parts := strings.SplitN(*holderIdentity, holderIdentitySeparator, 2) + id := &parts[0] + if len(parts) != 2 { + return id, nil + } + + switch druidv1alpha1.EtcdRole(parts[1]) { + case druidv1alpha1.EtcdRoleLeader: + role := druidv1alpha1.EtcdRoleLeader + return id, &role + case druidv1alpha1.EtcdRoleMember: + role := druidv1alpha1.EtcdRoleMember + return id, &role + default: + return id, nil + } +} + +func (r *readyCheck) checkContainersAreReady(ctx context.Context, namespace string, name string) (bool, error) { + pod := &corev1.Pod{} + if err := r.cl.Get(ctx, kutil.Key(namespace, name), pod); err != nil { + return false, err + } + + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.ContainersReady { + return cond.Status == corev1.ConditionTrue, nil + } + } + + return false, nil +} + +// ReadyCheck returns a check for the "Ready" condition. +func ReadyCheck(cl client.Client, logger logr.Logger, etcdMemberNotReadyThreshold, etcdMemberUnknownThreshold time.Duration) Checker { + return &readyCheck{ + logger: logger, + cl: cl, + etcdMemberNotReadyThreshold: etcdMemberNotReadyThreshold, + etcdMemberUnknownThreshold: etcdMemberUnknownThreshold, + } +} diff --git a/internal/health/etcdmember/check_ready_test.go b/internal/health/etcdmember/check_ready_test.go new file mode 100644 index 000000000..14e9bfbb9 --- /dev/null +++ b/internal/health/etcdmember/check_ready_test.go @@ -0,0 +1,403 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdmember_test + +import ( + "context" + "errors" + "fmt" + "time" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/go-logr/logr" + + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/test" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + . "github.com/gardener/etcd-druid/pkg/health/etcdmember" + mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + "github.com/gardener/etcd-druid/pkg/utils" +) + +var _ = Describe("ReadyCheck", func() { + Describe("#Check", func() { + var ( + ctx context.Context + mockCtrl *gomock.Controller + cl *mockclient.MockClient + leaseDurationSeconds *int32 + unknownThreshold, notReadyThreshold time.Duration + now time.Time + check Checker + logger logr.Logger + + member1Name string + member1ID *string + etcd druidv1alpha1.Etcd + leasesList *coordinationv1.LeaseList + ) + + BeforeEach(func() { + ctx = context.Background() + mockCtrl = gomock.NewController(GinkgoT()) + cl = mockclient.NewMockClient(mockCtrl) + unknownThreshold = 300 * time.Second + notReadyThreshold = 60 * time.Second + now, _ = time.Parse(time.RFC3339, "2021-06-01T00:00:00Z") + logger = log.Log.WithName("Test") + check = ReadyCheck(cl, logger, notReadyThreshold, unknownThreshold) + + member1ID = pointer.String("1") + member1Name = "member1" + + etcd = druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd", + Namespace: "etcd-test", + }, + } + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + JustBeforeEach(func() { + cl.EXPECT().List(ctx, gomock.AssignableToTypeOf(&coordinationv1.LeaseList{}), client.InNamespace(etcd.Namespace), + client.MatchingLabels{common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: utils.PurposeMemberLease}). + DoAndReturn( + func(_ context.Context, leases *coordinationv1.LeaseList, _ ...client.ListOption) error { + *leases = *leasesList + return nil + }) + }) + + Context("when just expired", func() { + BeforeEach(func() { + renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second)) + leasesList = &coordinationv1.LeaseList{ + Items: []coordinationv1.Lease{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: member1Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &renewTime, + }, + }, + }, + } + }) + + It("should set the affected condition to UNKNOWN because lease is lost", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { + *pod = corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.ContainersReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + return nil + }, + ) + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(1)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusUnknown)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + }) + + It("should set the affected condition to UNKNOWN because Pod cannot be received", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { + return errors.New("foo") + }, + ) + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(1)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusUnknown)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + }) + + It("should set the affected condition to FAILED because containers are not ready", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { + *pod = corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.ContainersReady, + Status: corev1.ConditionFalse, + }, + }, + }, + } + return nil + }, + ) + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(1)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusNotReady)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + }) + + It("should set the affected condition to FAILED because Pod is not found", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { + return apierrors.NewNotFound(corev1.Resource("pods"), member1Name) + }, + ) + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(1)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusNotReady)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + }) + }) + + Context("when expired a while ago", func() { + var ( + member2Name string + member2ID *string + ) + + BeforeEach(func() { + member2Name = "member2" + member2ID = pointer.String("2") + + var ( + shortExpirationTime = metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second)) + longExpirationTime = metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second).Add(-1 * notReadyThreshold)) + ) + + leasesList = &coordinationv1.LeaseList{ + Items: []coordinationv1.Lease{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: member1Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &shortExpirationTime, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: member2Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member2ID, druidv1alpha1.EtcdRoleMember)), + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &longExpirationTime, + }, + }, + }, + } + }) + + It("should set the affected condition to FAILED because status was Unknown for a while", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { + return errors.New("foo") + }, + ) + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(2)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusUnknown)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + Expect(results[1].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusNotReady)) + Expect(results[1].ID()).To(Equal(member2ID)) + Expect(results[1].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleMember))) + }) + }) + + Context("when lease is up-to-date", func() { + var ( + member2Name, member3Name string + member2ID, member3ID *string + ) + + BeforeEach(func() { + member2Name = "member2" + member2ID = pointer.String("2") + member3Name = "member3" + member3ID = pointer.String("3") + renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold)) + leasesList = &coordinationv1.LeaseList{ + Items: []coordinationv1.Lease{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: member1Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &renewTime, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: member2Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: member2ID, + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &renewTime, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: member3Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member3ID, "foo")), + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &renewTime, + }, + }, + }, + } + }) + + It("should set member ready", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(3)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + Expect(results[1].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) + Expect(results[1].ID()).To(Equal(member2ID)) + Expect(results[1].Role()).To(BeNil()) + Expect(results[2].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) + Expect(results[2].ID()).To(Equal(member3ID)) + Expect(results[2].Role()).To(BeNil()) + }) + }) + + Context("when lease has not been acquired", func() { + var ( + member2Name string + ) + + BeforeEach(func() { + member2Name = "member2" + renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold)) + leasesList = &coordinationv1.LeaseList{ + Items: []coordinationv1.Lease{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: member1Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), + LeaseDurationSeconds: leaseDurationSeconds, + RenewTime: &renewTime, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: member2Name, + Namespace: etcd.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.String("foo"), + LeaseDurationSeconds: leaseDurationSeconds, + }, + }, + }, + } + }) + + It("should only contain members which acquired lease once", func() { + defer test.WithVar(&TimeNow, func() time.Time { + return now + })() + + results := check.Check(ctx, etcd) + + Expect(results).To(HaveLen(1)) + Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) + Expect(results[0].ID()).To(Equal(member1ID)) + Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) + }) + }) + }) +}) diff --git a/internal/health/etcdmember/etcdmember_suite_test.go b/internal/health/etcdmember/etcdmember_suite_test.go new file mode 100644 index 000000000..ebba7f55a --- /dev/null +++ b/internal/health/etcdmember/etcdmember_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdmember_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestEtcdMember(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Etcd Member Suite") +} diff --git a/internal/health/etcdmember/types.go b/internal/health/etcdmember/types.go new file mode 100644 index 000000000..315e2aeb1 --- /dev/null +++ b/internal/health/etcdmember/types.go @@ -0,0 +1,63 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdmember + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +) + +// Checker is an interface to check the members of an etcd cluster. +type Checker interface { + Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Result +} + +// Result is an interface to capture the result of checks on etcd members. +type Result interface { + ID() *string + Name() string + Role() *druidv1alpha1.EtcdRole + Status() druidv1alpha1.EtcdMemberConditionStatus + Reason() string +} + +type result struct { + id *string + name string + role *druidv1alpha1.EtcdRole + status druidv1alpha1.EtcdMemberConditionStatus + reason string +} + +func (r *result) ID() *string { + return r.id +} + +func (r *result) Name() string { + return r.name +} + +func (r *result) Role() *druidv1alpha1.EtcdRole { + return r.role +} + +func (r *result) Status() druidv1alpha1.EtcdMemberConditionStatus { + return r.status +} + +func (r *result) Reason() string { + return r.reason +} diff --git a/internal/health/status/check.go b/internal/health/status/check.go new file mode 100644 index 000000000..e742f99c1 --- /dev/null +++ b/internal/health/status/check.go @@ -0,0 +1,149 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status + +import ( + "context" + "sync" + "time" + + "github.com/go-logr/logr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/health/condition" + "github.com/gardener/etcd-druid/pkg/health/etcdmember" +) + +// ConditionCheckFn is a type alias for a function which returns an implementation of `Check`. +type ConditionCheckFn func(client.Client) condition.Checker + +// EtcdMemberCheckFn is a type alias for a function which returns an implementation of `Check`. +type EtcdMemberCheckFn func(client.Client, logr.Logger, time.Duration, time.Duration) etcdmember.Checker + +// TimeNow is the function used to get the current time. +var TimeNow = time.Now + +var ( + // NewDefaultConditionBuilder is the default condition builder. + NewDefaultConditionBuilder = condition.NewBuilder + // NewDefaultEtcdMemberBuilder is the default etcd member builder. + NewDefaultEtcdMemberBuilder = etcdmember.NewBuilder + // ConditionChecks Checks are the registered condition checks. + ConditionChecks = []ConditionCheckFn{ + condition.AllMembersCheck, + condition.ReadyCheck, + condition.BackupReadyCheck, + } + // EtcdMemberChecks are the etcd member checks. + EtcdMemberChecks = []EtcdMemberCheckFn{ + etcdmember.ReadyCheck, + } +) + +// Checker checks Etcd status conditions and the status of the Etcd members. +type Checker struct { + cl client.Client + etcdMemberNotReadyThreshold time.Duration + etcdMemberUnknownThreshold time.Duration + conditionCheckFns []ConditionCheckFn + conditionBuilderFn func() condition.Builder + etcdMemberCheckFns []EtcdMemberCheckFn + etcdMemberBuilderFn func() etcdmember.Builder +} + +// Check executes the status checks and mutates the passed status object with the corresponding results. +func (c *Checker) Check(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { + // First execute the etcd member checks for the status. + if err := c.executeEtcdMemberChecks(ctx, logger, etcd); err != nil { + return err + } + + // Execute condition checks after the etcd member checks because we need their result here. + return c.executeConditionChecks(ctx, etcd) +} + +// executeConditionChecks runs all registered condition checks **in parallel**. +func (c *Checker) executeConditionChecks(ctx context.Context, etcd *druidv1alpha1.Etcd) error { + var ( + resultCh = make(chan condition.Result) + + wg sync.WaitGroup + ) + + // Run condition checks in parallel since they work independently from each other. + for _, newCheck := range c.conditionCheckFns { + c := newCheck(c.cl) + wg.Add(1) + go (func() { + defer wg.Done() + resultCh <- c.Check(ctx, *etcd) + })() + } + + go (func() { + defer close(resultCh) + wg.Wait() + })() + + results := make([]condition.Result, 0, len(ConditionChecks)) + for r := range resultCh { + results = append(results, r) + } + + conditions := c.conditionBuilderFn(). + WithNowFunc(func() metav1.Time { return metav1.NewTime(TimeNow()) }). + WithOldConditions(etcd.Status.Conditions). + WithResults(results). + Build(etcd.Spec.Replicas) + + etcd.Status.Conditions = conditions + return nil +} + +// executeEtcdMemberChecks runs all registered etcd member checks **sequentially**. +// The result of a check is passed via the `status` sub-resources to the next check. +func (c *Checker) executeEtcdMemberChecks(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { + // Run etcd member checks sequentially as most of them act on multiple elements. + for _, newCheck := range c.etcdMemberCheckFns { + results := newCheck(c.cl, logger, c.etcdMemberNotReadyThreshold, c.etcdMemberUnknownThreshold).Check(ctx, *etcd) + + // Build and assign the results after each check, so that the next check + // can act on the latest results. + memberStatuses := c.etcdMemberBuilderFn(). + WithNowFunc(func() metav1.Time { return metav1.NewTime(TimeNow()) }). + WithOldMembers(etcd.Status.Members). + WithResults(results). + Build() + + etcd.Status.Members = memberStatuses + } + return nil +} + +// NewChecker creates a new instance for checking the etcd status. +func NewChecker(cl client.Client, etcdMemberNotReadyThreshold, etcdMemberUnknownThreshold time.Duration) *Checker { + return &Checker{ + cl: cl, + etcdMemberNotReadyThreshold: etcdMemberNotReadyThreshold, + etcdMemberUnknownThreshold: etcdMemberUnknownThreshold, + conditionCheckFns: ConditionChecks, + conditionBuilderFn: NewDefaultConditionBuilder, + etcdMemberCheckFns: EtcdMemberChecks, + etcdMemberBuilderFn: NewDefaultEtcdMemberBuilder, + } +} diff --git a/internal/health/status/check_test.go b/internal/health/status/check_test.go new file mode 100644 index 000000000..f7dee7f65 --- /dev/null +++ b/internal/health/status/check_test.go @@ -0,0 +1,282 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status_test + +import ( + "context" + "time" + + "k8s.io/utils/pointer" + + "github.com/go-logr/logr" + + "github.com/gardener/etcd-druid/pkg/health/condition" + "github.com/gardener/etcd-druid/pkg/health/etcdmember" + "github.com/gardener/gardener/pkg/utils/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/health/status" +) + +var _ = Describe("Check", func() { + Describe("#Check", func() { + It("should correctly execute checks and fill status", func() { + memberRoleLeader := druidv1alpha1.EtcdRoleLeader + memberRoleMember := druidv1alpha1.EtcdRoleMember + + timeBefore, _ := time.Parse(time.RFC3339, "2021-06-01T00:00:00Z") + timeNow := timeBefore.Add(1 * time.Hour) + + status := druidv1alpha1.EtcdStatus{ + Conditions: []druidv1alpha1.Condition{ + { + Type: druidv1alpha1.ConditionTypeReady, + Status: druidv1alpha1.ConditionTrue, + LastTransitionTime: metav1.NewTime(timeBefore), + LastUpdateTime: metav1.NewTime(timeBefore), + Reason: "foo reason", + Message: "foo message", + }, + { + Type: druidv1alpha1.ConditionTypeAllMembersReady, + Status: druidv1alpha1.ConditionTrue, + LastTransitionTime: metav1.NewTime(timeBefore), + LastUpdateTime: metav1.NewTime(timeBefore), + Reason: "bar reason", + Message: "bar message", + }, + { + Type: druidv1alpha1.ConditionTypeBackupReady, + Status: druidv1alpha1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(timeBefore), + LastUpdateTime: metav1.NewTime(timeBefore), + Reason: "foobar reason", + Message: "foobar message", + }, + }, + Members: []druidv1alpha1.EtcdMemberStatus{ + { + ID: pointer.String("1"), + Name: "member1", + Status: druidv1alpha1.EtcdMemberStatusReady, + LastTransitionTime: metav1.NewTime(timeBefore), + Reason: "foo reason", + }, + { + ID: pointer.String("2"), + Name: "member2", + Status: druidv1alpha1.EtcdMemberStatusNotReady, + LastTransitionTime: metav1.NewTime(timeBefore), + Reason: "bar reason", + }, + { + ID: pointer.String("3"), + Name: "member3", + Status: druidv1alpha1.EtcdMemberStatusReady, + LastTransitionTime: metav1.NewTime(timeBefore), + Reason: "foobar reason", + }, + }, + } + + etcd := &druidv1alpha1.Etcd{ + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 1, + }, + Status: status, + } + + defer test.WithVar(&ConditionChecks, []ConditionCheckFn{ + func(client.Client) condition.Checker { + return createConditionCheck(druidv1alpha1.ConditionTypeReady, druidv1alpha1.ConditionFalse, "FailedConditionCheck", "check failed") + }, + func(client.Client) condition.Checker { + return createConditionCheck(druidv1alpha1.ConditionTypeAllMembersReady, druidv1alpha1.ConditionTrue, "bar reason", "bar message") + }, + func(client.Client) condition.Checker { + return createConditionCheck(druidv1alpha1.ConditionTypeBackupReady, druidv1alpha1.ConditionUnknown, "foobar reason", "foobar message") + }, + })() + + defer test.WithVar(&EtcdMemberChecks, []EtcdMemberCheckFn{ + func(_ client.Client, _ logr.Logger, _, _ time.Duration) etcdmember.Checker { + return createEtcdMemberCheck( + etcdMemberResult{pointer.String("1"), "member1", &memberRoleLeader, druidv1alpha1.EtcdMemberStatusUnknown, "Unknown"}, + etcdMemberResult{pointer.String("2"), "member2", &memberRoleMember, druidv1alpha1.EtcdMemberStatusNotReady, "bar reason"}, + etcdMemberResult{pointer.String("3"), "member3", &memberRoleMember, druidv1alpha1.EtcdMemberStatusReady, "foobar reason"}, + ) + }, + })() + + defer test.WithVar(&TimeNow, func() time.Time { return timeNow })() + + checker := NewChecker(nil, 5*time.Minute, time.Minute) + logger := log.Log.WithName("Test") + + Expect(checker.Check(context.Background(), logger, etcd)).To(Succeed()) + + Expect(etcd.Status.Conditions).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeReady), + "Status": Equal(druidv1alpha1.ConditionFalse), + "LastTransitionTime": Equal(metav1.NewTime(timeNow)), + "LastUpdateTime": Equal(metav1.NewTime(timeNow)), + "Reason": Equal("FailedConditionCheck"), + "Message": Equal("check failed"), + }), + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeAllMembersReady), + "Status": Equal(druidv1alpha1.ConditionTrue), + "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), + "LastUpdateTime": Equal(metav1.NewTime(timeNow)), + "Reason": Equal("bar reason"), + "Message": Equal("bar message"), + }), + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(druidv1alpha1.ConditionTypeBackupReady), + "Status": Equal(druidv1alpha1.ConditionUnknown), + "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), + "LastUpdateTime": Equal(metav1.NewTime(timeNow)), + "Reason": Equal("foobar reason"), + "Message": Equal("foobar message"), + }), + )) + + Expect(etcd.Status.Members).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "ID": PointTo(Equal("1")), + "Name": Equal("member1"), + "Role": PointTo(Equal(druidv1alpha1.EtcdRoleLeader)), + "Status": Equal(druidv1alpha1.EtcdMemberStatusUnknown), + "LastTransitionTime": Equal(metav1.NewTime(timeNow)), + "Reason": Equal("Unknown"), + }), + MatchFields(IgnoreExtras, Fields{ + "ID": PointTo(Equal("2")), + "Name": Equal("member2"), + "Role": PointTo(Equal(druidv1alpha1.EtcdRoleMember)), + "Status": Equal(druidv1alpha1.EtcdMemberStatusNotReady), + "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), + "Reason": Equal("bar reason"), + }), + MatchFields(IgnoreExtras, Fields{ + "ID": PointTo(Equal("3")), + "Name": Equal("member3"), + "Role": PointTo(Equal(druidv1alpha1.EtcdRoleMember)), + "Status": Equal(druidv1alpha1.EtcdMemberStatusReady), + "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), + "Reason": Equal("foobar reason"), + }), + )) + + }) + }) +}) + +type conditionResult struct { + ConType druidv1alpha1.ConditionType + ConStatus druidv1alpha1.ConditionStatus + ConReason string + ConMessage string +} + +func (r *conditionResult) ConditionType() druidv1alpha1.ConditionType { + return r.ConType +} + +func (r *conditionResult) Status() druidv1alpha1.ConditionStatus { + return r.ConStatus +} + +func (r *conditionResult) Reason() string { + return r.ConReason +} + +func (r *conditionResult) Message() string { + return r.ConMessage +} + +type testChecker struct { + result *conditionResult +} + +func (t *testChecker) Check(_ context.Context, _ druidv1alpha1.Etcd) condition.Result { + return t.result +} + +func createConditionCheck(conType druidv1alpha1.ConditionType, status druidv1alpha1.ConditionStatus, reason, message string) condition.Checker { + return &testChecker{ + result: &conditionResult{ + ConType: conType, + ConStatus: status, + ConReason: reason, + ConMessage: message, + }, + } +} + +type etcdMemberResult struct { + id *string + name string + role *druidv1alpha1.EtcdRole + status druidv1alpha1.EtcdMemberConditionStatus + reason string +} + +func (r *etcdMemberResult) ID() *string { + return r.id +} + +func (r *etcdMemberResult) Name() string { + return r.name +} + +func (r *etcdMemberResult) Role() *druidv1alpha1.EtcdRole { + return r.role +} + +func (r *etcdMemberResult) Status() druidv1alpha1.EtcdMemberConditionStatus { + return r.status +} + +func (r *etcdMemberResult) Reason() string { + return r.reason +} + +type etcdMemberTestChecker struct { + results []etcdMemberResult +} + +func (t *etcdMemberTestChecker) Check(_ context.Context, _ druidv1alpha1.Etcd) []etcdmember.Result { + var results []etcdmember.Result + for _, r := range t.results { + result := r + results = append(results, &result) + } + + return results +} + +func createEtcdMemberCheck(results ...etcdMemberResult) etcdmember.Checker { + return &etcdMemberTestChecker{ + results: results, + } +} diff --git a/internal/health/status/status_suite_test.go b/internal/health/status/status_suite_test.go new file mode 100644 index 000000000..79ac82d0e --- /dev/null +++ b/internal/health/status/status_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestStatus(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Status Suite") +} diff --git a/internal/mapper/etcd_to_secret.go b/internal/mapper/etcd_to_secret.go new file mode 100644 index 000000000..b705d2fda --- /dev/null +++ b/internal/mapper/etcd_to_secret.go @@ -0,0 +1,82 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/go-logr/logr" + + "github.com/gardener/gardener/pkg/controllerutils/mapper" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type etcdToSecretMapper struct{} + +func (m *etcdToSecretMapper) Map(_ context.Context, _ logr.Logger, _ client.Reader, obj client.Object) []reconcile.Request { + etcd, ok := obj.(*druidv1alpha1.Etcd) + if !ok { + return nil + } + + var requests []reconcile.Request + + if etcd.Spec.Etcd.ClientUrlTLS != nil { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, + }}) + + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, + }}) + + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, + }}) + } + + if etcd.Spec.Etcd.PeerUrlTLS != nil { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, + }}) + + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, + }}) + } + + if etcd.Spec.Backup.Store != nil && etcd.Spec.Backup.Store.SecretRef != nil { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Spec.Backup.Store.SecretRef.Name, + }}) + } + + return requests +} + +// EtcdToSecret returns a mapper that returns a request for the Secret resources +// referenced in the Etcd for which an event happened. +func EtcdToSecret() mapper.Mapper { + return &etcdToSecretMapper{} +} diff --git a/internal/mapper/etcd_to_secret_test.go b/internal/mapper/etcd_to_secret_test.go new file mode 100644 index 000000000..63a8ec977 --- /dev/null +++ b/internal/mapper/etcd_to_secret_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper_test + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/pkg/mapper" + "github.com/go-logr/logr" + + "github.com/gardener/gardener/pkg/controllerutils/mapper" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("EtcdToSecret", func() { + var ( + ctx = context.Background() + m mapper.Mapper + etcd *druidv1alpha1.Etcd + logger logr.Logger + + namespace = "some-namespace" + ) + + BeforeEach(func() { + m = EtcdToSecret() + + etcd = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + }, + } + logger = log.Log.WithName("Test") + }) + + It("should return empty list because Etcd is not referencing secrets", func() { + Expect(m.Map(ctx, logger, nil, etcd)).To(BeEmpty()) + }) + + It("should return four requests because Etcd is referencing secrets", func() { + var ( + secretClientCATLS = "client-url-ca-etcd" + secretClientServerTLS = "client-url-etcd-server-tls" + secretClientClientTLS = "client-url-etcd-client-tls" + secretPeerCATLS = "peer-url-ca-etcd" + secretPeerServerTLS = "peer-url-etcd-server-tls" + secretBackupStore = "backup-store" + ) + + etcd.Spec.Etcd.ClientUrlTLS = &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{Name: secretClientCATLS}, + }, + ServerTLSSecretRef: corev1.SecretReference{Name: secretClientServerTLS}, + ClientTLSSecretRef: corev1.SecretReference{Name: secretClientClientTLS}, + } + + etcd.Spec.Etcd.PeerUrlTLS = &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{Name: secretPeerCATLS}, + }, + ServerTLSSecretRef: corev1.SecretReference{Name: secretPeerServerTLS}, + } + + etcd.Spec.Backup.Store = &druidv1alpha1.StoreSpec{ + SecretRef: &corev1.SecretReference{Name: secretBackupStore}, + } + + Expect(m.Map(ctx, logger, nil, etcd)).To(ConsistOf( + reconcile.Request{NamespacedName: types.NamespacedName{ + Name: secretClientCATLS, + Namespace: namespace, + }}, + reconcile.Request{NamespacedName: types.NamespacedName{ + Name: secretClientServerTLS, + Namespace: namespace, + }}, + reconcile.Request{NamespacedName: types.NamespacedName{ + Name: secretClientClientTLS, + Namespace: namespace, + }}, + reconcile.Request{NamespacedName: types.NamespacedName{ + Name: secretPeerCATLS, + Namespace: namespace, + }}, + reconcile.Request{NamespacedName: types.NamespacedName{ + Name: secretPeerServerTLS, + Namespace: namespace, + }}, + reconcile.Request{NamespacedName: types.NamespacedName{ + Name: secretBackupStore, + Namespace: namespace, + }}, + )) + }) +}) diff --git a/internal/mapper/mapper_suite_test.go b/internal/mapper/mapper_suite_test.go new file mode 100644 index 000000000..4cf5eed2c --- /dev/null +++ b/internal/mapper/mapper_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMapper(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Mapper Suite") +} diff --git a/internal/mapper/statefulset_to_etcd.go b/internal/mapper/statefulset_to_etcd.go new file mode 100644 index 000000000..0a3c81713 --- /dev/null +++ b/internal/mapper/statefulset_to_etcd.go @@ -0,0 +1,76 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/gardener/pkg/controllerutils/mapper" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/gardener/etcd-druid/pkg/common" +) + +type statefulSetToEtcdMapper struct { + ctx context.Context + cl client.Client +} + +func (m *statefulSetToEtcdMapper) Map(_ context.Context, _ logr.Logger, _ client.Reader, obj client.Object) []reconcile.Request { + sts, ok := obj.(*appsv1.StatefulSet) + if !ok { + return nil + } + + ownerKey, ok := sts.Annotations[common.GardenerOwnedBy] + if !ok { + return nil + } + + name, namespace, err := cache.SplitMetaNamespaceKey(ownerKey) + if err != nil { + return nil + } + + etcd := &druidv1alpha1.Etcd{} + if err := m.cl.Get(m.ctx, kutil.Key(name, namespace), etcd); err != nil { + return nil + } + + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: etcd.Namespace, + Name: etcd.Name, + }, + }, + } +} + +// StatefulSetToEtcd returns a mapper that returns a request for the Etcd resource +// that owns the StatefulSet for which an event happened. +func StatefulSetToEtcd(ctx context.Context, cl client.Client) mapper.Mapper { + return &statefulSetToEtcdMapper{ + ctx: ctx, + cl: cl, + } +} diff --git a/internal/mapper/statefulset_to_etcd_test.go b/internal/mapper/statefulset_to_etcd_test.go new file mode 100644 index 000000000..6c6ce84ce --- /dev/null +++ b/internal/mapper/statefulset_to_etcd_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper_test + +import ( + "context" + "fmt" + + "github.com/gardener/gardener/pkg/controllerutils/mapper" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + . "github.com/gardener/etcd-druid/pkg/mapper" + mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" +) + +var _ = Describe("Druid Mapper", func() { + var ( + ctx = context.Background() + ctrl *gomock.Controller + c *mockclient.MockClient + logger logr.Logger + + name, namespace, key string + statefulset *appsv1.StatefulSet + mapper mapper.Mapper + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + c = mockclient.NewMockClient(ctrl) + logger = log.Log.WithName("Test") + + name = "etcd-test" + namespace = "test" + key = fmt.Sprintf("%s/%s", namespace, name) + statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + mapper = StatefulSetToEtcd(ctx, c) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#StatefulSetToEtcd", func() { + It("should find related Etcd object", func() { + etcd := &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + c.EXPECT().Get(ctx, kutil.Key(namespace, name), gomock.AssignableToTypeOf(etcd), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectKey, obj *druidv1alpha1.Etcd, _ ...client.GetOption) error { + *obj = *etcd + return nil + }, + ) + + kutil.SetMetaDataAnnotation(statefulset, common.GardenerOwnedBy, key) + + etcds := mapper.Map(ctx, logger, nil, statefulset) + + Expect(etcds).To(ConsistOf( + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: namespace, + Name: name, + }, + }, + )) + }) + + It("should not find related Etcd object because an error occurred during retrieval", func() { + c.EXPECT().Get(ctx, kutil.Key(namespace, name), gomock.AssignableToTypeOf(&druidv1alpha1.Etcd{})).Return(fmt.Errorf("foo error")) + + kutil.SetMetaDataAnnotation(statefulset, common.GardenerOwnedBy, key) + + etcds := mapper.Map(ctx, logger, nil, statefulset) + + Expect(etcds).To(BeEmpty()) + }) + + It("should not find related Etcd object because owner annotation is not present", func() { + etcds := mapper.Map(ctx, logger, nil, statefulset) + + Expect(etcds).To(BeEmpty()) + }) + + It("should not find related Etcd object because map is called with wrong object", func() { + etcds := mapper.Map(ctx, logger, nil, nil) + + Expect(etcds).To(BeEmpty()) + }) + }) +}) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 000000000..3c02c5b4f --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,120 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "sort" + +const ( + // LabelSucceeded is a metric label indicating whether associated metric + // series is for success or failure. + LabelSucceeded = "succeeded" + // ValueSucceededTrue is value True for metric label succeeded. + ValueSucceededTrue = "true" + // ValueSucceededFalse is value False for metric label failed. + ValueSucceededFalse = "false" + + // EtcdNamespace is the label for prometheus metrics to indicate etcd namespace + EtcdNamespace = "etcd_namespace" +) + +var ( + // DruidLabels are the labels for prometheus metrics + DruidLabels = map[string][]string{ + LabelSucceeded: { + ValueSucceededFalse, + ValueSucceededTrue, + }, + } +) + +// GenerateLabelCombinations generates combinations of label values for metrics +func GenerateLabelCombinations(labelValues map[string][]string) []map[string]string { + labels := make([]string, len(labelValues)) + valuesList := make([][]string, len(labelValues)) + valueCounts := make([]int, len(labelValues)) + i := 0 + for label := range labelValues { + labels[i] = label + i++ + } + sort.Strings(labels) + for i, label := range labels { + values := make([]string, len(labelValues[label])) + copy(values, labelValues[label]) + valuesList[i] = values + valueCounts[i] = len(values) + } + combinations := getCombinations(valuesList) + + output := make([]map[string]string, len(combinations)) + for i, combination := range combinations { + labelVals := make(map[string]string, len(labels)) + for j := 0; j < len(labels); j++ { + labelVals[labels[j]] = combination[j] + } + output[i] = labelVals + } + return output +} + +// getCombinations returns combinations of slice of string slices +func getCombinations(valuesList [][]string) [][]string { + if len(valuesList) == 0 { + return [][]string{} + } else if len(valuesList) == 1 { + return wrapInSlice(valuesList[0]) + } + + return cartesianProduct(wrapInSlice(valuesList[0]), getCombinations(valuesList[1:])) +} + +// cartesianProduct combines two slices of slice of strings while also +// combining the sub-slices of strings into a single string +// Ex: +// a => [[p,q],[r,s]] +// b => [[1,2],[3,4]] +// Output => [[p,q,1,2],[p,q,3,4],[r,s,1,2],[r,s,3,4]] +func cartesianProduct(a [][]string, b [][]string) [][]string { + output := make([][]string, len(a)*len(b)) + for i := 0; i < len(a); i++ { + for j := 0; j < len(b); j++ { + arr := make([]string, len(a[i])+len(b[j])) + ctr := 0 + for ii := 0; ii < len(a[i]); ii++ { + arr[ctr] = a[i][ii] + ctr++ + } + for jj := 0; jj < len(b[j]); jj++ { + arr[ctr] = b[j][jj] + ctr++ + } + output[(i*len(b))+j] = arr + } + } + return output +} + +// wrapInSlice is a helper function to wrap a slice of strings within +// a slice of slices of strings +// Ex: [p,q,r] -> [[p],[q],[r]] +func wrapInSlice(s []string) [][]string { + output := make([][]string, len(s)) + for i := 0; i < len(output); i++ { + elem := make([]string, 1) + elem[0] = s[i] + output[i] = elem + } + return output +} diff --git a/internal/metrics/metrics_suite_test.go b/internal/metrics/metrics_suite_test.go new file mode 100644 index 000000000..506884539 --- /dev/null +++ b/internal/metrics/metrics_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Suite") +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 000000000..1e8de70d8 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,90 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Metrics", func() { + Describe("Testing helper functions for metrics initialization", func() { + Context("Testing wrapInSlice with input", func() { + It("should return expected output", func() { + input := []string{"a", "b", "c"} + expectedOutput := [][]string{{"a"}, {"b"}, {"c"}} + output := wrapInSlice(input) + Expect(output).Should(Equal(expectedOutput)) + }) + }) + Context("Testing cartesianProduct with inputs", func() { + It("should return expected output", func() { + input1 := [][]string{{"p", "q"}, {"r", "s"}} + input2 := [][]string{{"1", "2"}, {"3", "4"}} + expectedOutput := [][]string{ + {"p", "q", "1", "2"}, + {"p", "q", "3", "4"}, + {"r", "s", "1", "2"}, + {"r", "s", "3", "4"}, + } + output := cartesianProduct(input1, input2) + Expect(output).Should(Equal(expectedOutput)) + }) + }) + Context("Testing generateLabelCombinations with input of one label", func() { + It("should return expected output", func() { + input := map[string][]string{ + "a": { + "1", + "2", + "3", + }, + } + expectedOutput := []map[string]string{ + {"a": "1"}, + {"a": "2"}, + {"a": "3"}, + } + output := GenerateLabelCombinations(input) + Expect(output).Should(Equal(expectedOutput)) + }) + }) + Context("Testing generateLabelCombinations with input of two labels", func() { + It("should return expected output", func() { + input := map[string][]string{ + "a": { + "1", + "2", + "3", + }, + "b": { + "4", + "5", + }, + } + expectedOutput := []map[string]string{ + {"a": "1", "b": "4"}, + {"a": "1", "b": "5"}, + {"a": "2", "b": "4"}, + {"a": "2", "b": "5"}, + {"a": "3", "b": "4"}, + {"a": "3", "b": "5"}, + } + output := GenerateLabelCombinations(input) + Expect(output).Should(Equal(expectedOutput)) + }) + }) + }) +}) diff --git a/internal/mock/controller-runtime/client/doc.go b/internal/mock/controller-runtime/client/doc.go new file mode 100644 index 000000000..fa2f38c39 --- /dev/null +++ b/internal/mock/controller-runtime/client/doc.go @@ -0,0 +1,16 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:generate mockgen -package client -destination=mocks.go sigs.k8s.io/controller-runtime/pkg/client Client,StatusWriter,Reader,Writer + +package client diff --git a/internal/mock/controller-runtime/client/mocks.go b/internal/mock/controller-runtime/client/mocks.go new file mode 100644 index 000000000..a43241564 --- /dev/null +++ b/internal/mock/controller-runtime/client/mocks.go @@ -0,0 +1,487 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: sigs.k8s.io/controller-runtime/pkg/client (interfaces: Client,StatusWriter,Reader,Writer) + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + meta "k8s.io/apimachinery/pkg/api/meta" + runtime "k8s.io/apimachinery/pkg/runtime" + types "k8s.io/apimachinery/pkg/types" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockClient) Create(arg0 context.Context, arg1 client.Object, arg2 ...client.CreateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Create", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockClientMockRecorder) Create(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockClient)(nil).Create), varargs...) +} + +// Delete mocks base method. +func (m *MockClient) Delete(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Delete", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockClientMockRecorder) Delete(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockClient)(nil).Delete), varargs...) +} + +// DeleteAllOf mocks base method. +func (m *MockClient) DeleteAllOf(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteAllOfOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteAllOf", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllOf indicates an expected call of DeleteAllOf. +func (mr *MockClientMockRecorder) DeleteAllOf(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockClient)(nil).DeleteAllOf), varargs...) +} + +// Get mocks base method. +func (m *MockClient) Get(arg0 context.Context, arg1 types.NamespacedName, arg2 client.Object, arg3 ...client.GetOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Get", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Get indicates an expected call of Get. +func (mr *MockClientMockRecorder) Get(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), varargs...) +} + +// List mocks base method. +func (m *MockClient) List(arg0 context.Context, arg1 client.ObjectList, arg2 ...client.ListOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "List", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockClientMockRecorder) List(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockClient)(nil).List), varargs...) +} + +// Patch mocks base method. +func (m *MockClient) Patch(arg0 context.Context, arg1 client.Object, arg2 client.Patch, arg3 ...client.PatchOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockClientMockRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockClient)(nil).Patch), varargs...) +} + +// RESTMapper mocks base method. +func (m *MockClient) RESTMapper() meta.RESTMapper { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RESTMapper") + ret0, _ := ret[0].(meta.RESTMapper) + return ret0 +} + +// RESTMapper indicates an expected call of RESTMapper. +func (mr *MockClientMockRecorder) RESTMapper() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RESTMapper", reflect.TypeOf((*MockClient)(nil).RESTMapper)) +} + +// Scheme mocks base method. +func (m *MockClient) Scheme() *runtime.Scheme { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Scheme") + ret0, _ := ret[0].(*runtime.Scheme) + return ret0 +} + +// Scheme indicates an expected call of Scheme. +func (mr *MockClientMockRecorder) Scheme() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Scheme", reflect.TypeOf((*MockClient)(nil).Scheme)) +} + +// Status mocks base method. +func (m *MockClient) Status() client.SubResourceWriter { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status") + ret0, _ := ret[0].(client.SubResourceWriter) + return ret0 +} + +// Status indicates an expected call of Status. +func (mr *MockClientMockRecorder) Status() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockClient)(nil).Status)) +} + +// SubResource mocks base method. +func (m *MockClient) SubResource(arg0 string) client.SubResourceClient { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SubResource", arg0) + ret0, _ := ret[0].(client.SubResourceClient) + return ret0 +} + +// SubResource indicates an expected call of SubResource. +func (mr *MockClientMockRecorder) SubResource(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubResource", reflect.TypeOf((*MockClient)(nil).SubResource), arg0) +} + +// Update mocks base method. +func (m *MockClient) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.UpdateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockClientMockRecorder) Update(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockClient)(nil).Update), varargs...) +} + +// MockStatusWriter is a mock of StatusWriter interface. +type MockStatusWriter struct { + ctrl *gomock.Controller + recorder *MockStatusWriterMockRecorder +} + +// MockStatusWriterMockRecorder is the mock recorder for MockStatusWriter. +type MockStatusWriterMockRecorder struct { + mock *MockStatusWriter +} + +// NewMockStatusWriter creates a new mock instance. +func NewMockStatusWriter(ctrl *gomock.Controller) *MockStatusWriter { + mock := &MockStatusWriter{ctrl: ctrl} + mock.recorder = &MockStatusWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStatusWriter) EXPECT() *MockStatusWriterMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockStatusWriter) Create(arg0 context.Context, arg1, arg2 client.Object, arg3 ...client.SubResourceCreateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Create", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockStatusWriterMockRecorder) Create(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockStatusWriter)(nil).Create), varargs...) +} + +// Patch mocks base method. +func (m *MockStatusWriter) Patch(arg0 context.Context, arg1 client.Object, arg2 client.Patch, arg3 ...client.SubResourcePatchOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockStatusWriterMockRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockStatusWriter)(nil).Patch), varargs...) +} + +// Update mocks base method. +func (m *MockStatusWriter) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.SubResourceUpdateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockStatusWriterMockRecorder) Update(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockStatusWriter)(nil).Update), varargs...) +} + +// MockReader is a mock of Reader interface. +type MockReader struct { + ctrl *gomock.Controller + recorder *MockReaderMockRecorder +} + +// MockReaderMockRecorder is the mock recorder for MockReader. +type MockReaderMockRecorder struct { + mock *MockReader +} + +// NewMockReader creates a new mock instance. +func NewMockReader(ctrl *gomock.Controller) *MockReader { + mock := &MockReader{ctrl: ctrl} + mock.recorder = &MockReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockReader) EXPECT() *MockReaderMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockReader) Get(arg0 context.Context, arg1 types.NamespacedName, arg2 client.Object, arg3 ...client.GetOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Get", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Get indicates an expected call of Get. +func (mr *MockReaderMockRecorder) Get(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockReader)(nil).Get), varargs...) +} + +// List mocks base method. +func (m *MockReader) List(arg0 context.Context, arg1 client.ObjectList, arg2 ...client.ListOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "List", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockReaderMockRecorder) List(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockReader)(nil).List), varargs...) +} + +// MockWriter is a mock of Writer interface. +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter. +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance. +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockWriter) Create(arg0 context.Context, arg1 client.Object, arg2 ...client.CreateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Create", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockWriterMockRecorder) Create(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockWriter)(nil).Create), varargs...) +} + +// Delete mocks base method. +func (m *MockWriter) Delete(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Delete", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockWriterMockRecorder) Delete(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockWriter)(nil).Delete), varargs...) +} + +// DeleteAllOf mocks base method. +func (m *MockWriter) DeleteAllOf(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteAllOfOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteAllOf", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllOf indicates an expected call of DeleteAllOf. +func (mr *MockWriterMockRecorder) DeleteAllOf(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockWriter)(nil).DeleteAllOf), varargs...) +} + +// Patch mocks base method. +func (m *MockWriter) Patch(arg0 context.Context, arg1 client.Object, arg2 client.Patch, arg3 ...client.PatchOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockWriterMockRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockWriter)(nil).Patch), varargs...) +} + +// Update mocks base method. +func (m *MockWriter) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.UpdateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockWriterMockRecorder) Update(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockWriter)(nil).Update), varargs...) +} diff --git a/internal/resource/clientservice/clientservice.go b/internal/resource/clientservice/clientservice.go new file mode 100644 index 000000000..459d63c0d --- /dev/null +++ b/internal/resource/clientservice/clientservice.go @@ -0,0 +1,144 @@ +package clientservice + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// default values +const ( + defaultBackupPort = 8080 + defaultClientPort = 2379 + defaultServerPort = 2380 +) + +const ( + ErrDeletingClientService druidv1alpha1.ErrorCode = "ERR_DELETING_CLIENT_SERVICE" + ErrSyncingClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" +) + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + svc := &corev1.Service{} + if err := r.client.Get(ctx, getObjectKey(etcd), svc); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, svc.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + svc := emptyClientService(getObjectKey(etcd)) + _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { + svc.Labels = getLabels(etcd) + svc.Annotations = getAnnotations(etcd) + svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Spec.SessionAffinity = corev1.ServiceAffinityNone + svc.Spec.Selector = etcd.GetDefaultLabels() + svc.Spec.Ports = getPorts(etcd) + + return nil + }) + return druiderr.WrapError(err, + ErrSyncingClientService, + "Sync", + "Error during create or update of client service", + ) +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + objectKey := getObjectKey(etcd) + r.logger.Info("Triggering delete of client service", "objectKey", objectKey) + err := client.IgnoreNotFound(r.client.Delete(ctx, emptyClientService(objectKey))) + return druiderr.WrapError( + err, + ErrDeletingClientService, + "TriggerDelete", + "Failed to delete client service", + ) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{ + Name: etcd.GetClientServiceName(), + Namespace: etcd.Namespace, + } +} + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + var labelMaps []map[string]string + labelMaps = append(labelMaps, etcd.GetDefaultLabels()) + if etcd.Spec.Etcd.ClientService != nil { + labelMaps = append(labelMaps, etcd.Spec.Etcd.ClientService.Labels) + } + return utils.MergeMaps[string, string](labelMaps...) +} + +func getAnnotations(etcd *druidv1alpha1.Etcd) map[string]string { + if etcd.Spec.Etcd.ClientService != nil { + return etcd.Spec.Etcd.ClientService.Annotations + } + return nil +} + +func emptyClientService(objectKey client.ObjectKey) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} + +func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + + return []corev1.ServicePort{ + { + Name: "client", + Protocol: corev1.ProtocolTCP, + Port: clientPort, + TargetPort: intstr.FromInt(int(clientPort)), + }, + // TODO: Remove the "server" port in a future release + { + Name: "server", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }, + { + Name: "backuprestore", + Protocol: corev1.ProtocolTCP, + Port: backupPort, + TargetPort: intstr.FromInt(int(backupPort)), + }, + } +} diff --git a/internal/resource/configmap/configmap.go b/internal/resource/configmap/configmap.go new file mode 100644 index 000000000..5504a3b3b --- /dev/null +++ b/internal/resource/configmap/configmap.go @@ -0,0 +1,97 @@ +package configmap + +import ( + "encoding/json" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/utils" + "github.com/go-logr/logr" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + cm := &corev1.ConfigMap{} + if err := r.client.Get(ctx, getObjectKey(etcd), cm); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, cm.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + cfg := createEtcdConfig(etcd) + cfgYaml, err := yaml.Marshal(cfg) + if err != nil { + return err + } + cm := emptyConfigMap(getObjectKey(etcd)) + _, err = controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { + cm.Name = etcd.GetConfigMapName() + cm.Namespace = etcd.Namespace + cm.Labels = etcd.GetDefaultLabels() + cm.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + cm.Data = map[string]string{"etcd.conf.yaml": string(cfgYaml)} + return nil + }) + if err != nil { + return err + } + checkSum, err := computeCheckSum(cm) + if err != nil { + return err + } + ctx.Data[resource.ConfigMapCheckSumKey] = checkSum + return nil +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + objectKey := getObjectKey(etcd) + r.logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) + return client.IgnoreNotFound(r.client.Delete(ctx, emptyConfigMap(objectKey))) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{ + Name: etcd.GetConfigMapName(), + Namespace: etcd.Namespace, + } +} + +func emptyConfigMap(objectKey client.ObjectKey) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} + +func computeCheckSum(cm *corev1.ConfigMap) (string, error) { + jsonData, err := json.Marshal(cm.Data) + if err != nil { + return "", nil + } + return utils.ComputeSHA256Hex(jsonData), nil +} diff --git a/internal/resource/configmap/etcdconfig.go b/internal/resource/configmap/etcdconfig.go new file mode 100644 index 000000000..aec6e21bf --- /dev/null +++ b/internal/resource/configmap/etcdconfig.go @@ -0,0 +1,125 @@ +package configmap + +import ( + "fmt" + "strconv" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" + "k8s.io/utils/pointer" +) + +// default values +const ( + defaultDBQuotaBytes = int64(8 * 1024 * 1024 * 1024) // 8Gi + defaultAutoCompactionRetention = "30m" + defaultDataDir = "/var/etcd/data/new.etcd" + defaultInitialClusterToken = "etcd-cluster" + defaultInitialClusterState = "new" + // For more information refer to https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention + // TODO: Ideally this should be made configurable via Etcd resource as this has a direct impact on the memory requirements for etcd container. + // which in turn is influenced by the size of objects that are getting stored in etcd. + defaultSnapshotCount = 75000 + defaultClientPort = 2379 + defaultServerPort = 2380 +) + +type tlsTarget string + +const ( + clientTLS tlsTarget = "client" + peerTLS tlsTarget = "peer" +) + +type etcdConfig struct { + Name string `yaml:"name"` + DataDir string `yaml:"data-dir"` + Metrics druidv1alpha1.MetricsLevel `yaml:"metrics"` + SnapshotCount int `yaml:"snapshot-count"` + EnableV2 bool `yaml:"enable-v2"` + QuotaBackendBytes int64 `yaml:"quota-backend-bytes"` + InitialClusterToken string `yaml:"initial-cluster-token"` + InitialClusterState string `yaml:"initial-cluster-state"` + InitialCluster string `yaml:"initial-cluster"` + AutoCompactionMode druidv1alpha1.CompactionMode `yaml:"auto-compaction-mode"` + AutoCompactionRetention string `yaml:"auto-compaction-retention"` + ListenPeerUrls string `yaml:"listen-peer-urls"` + ListenClientUrls string `yaml:"listen-client-urls"` + AdvertisePeerUrls string `yaml:"initial-advertise-peer-urls"` + AdvertiseClientUrls string `yaml:"advertise-client-urls"` + ClientSecurity securityConfig `yaml:"client-transport-security,omitempty"` + PeerSecurity securityConfig `yaml:"peer-transport-security,omitempty"` +} + +type securityConfig struct { + CertFile string `yaml:"cert-file,omitempty"` + KeyFile string `yaml:"key-file,omitempty"` + ClientCertAuth bool `yaml:"client-cert-auth,omitempty"` + TrustedCAFile string `yaml:"trusted-ca-file,omitempty"` + AutoTLS bool `yaml:"auto-tls"` +} + +func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { + peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, peerTLS) + clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, clientTLS) + + cfg := &etcdConfig{ + Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), + DataDir: defaultDataDir, + Metrics: utils.TypeDeref[druidv1alpha1.MetricsLevel](etcd.Spec.Etcd.Metrics, druidv1alpha1.Basic), + SnapshotCount: defaultSnapshotCount, + EnableV2: false, + QuotaBackendBytes: getDBQuotaBytes(etcd), + InitialClusterToken: defaultInitialClusterToken, + InitialClusterState: defaultInitialClusterState, + InitialCluster: prepareInitialCluster(etcd, peerScheme), + AutoCompactionMode: utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), + AutoCompactionRetention: utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), + ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), + ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), + AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), + AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), + } + if peerSecurityConfig != nil { + cfg.PeerSecurity = *peerSecurityConfig + } + if clientSecurityConfig != nil { + cfg.ClientSecurity = *clientSecurityConfig + } + + return cfg +} + +func getDBQuotaBytes(etcd *druidv1alpha1.Etcd) int64 { + dbQuotaBytes := defaultDBQuotaBytes + if etcd.Spec.Etcd.Quota != nil { + dbQuotaBytes = etcd.Spec.Etcd.Quota.Value() + } + return dbQuotaBytes +} + +func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, tlsTarget tlsTarget) (string, *securityConfig) { + if tlsConfig != nil { + const defaultTLSCASecretKey = "ca.crt" + return "https", &securityConfig{ + CertFile: fmt.Sprintf("/var/etcd/ssl/%s/server/tls.crt", tlsTarget), + KeyFile: fmt.Sprintf("/var/etcd/ssl/%s/server/tls.key", tlsTarget), + ClientCertAuth: true, + TrustedCAFile: fmt.Sprintf("/var/etcd/ssl/%s/ca/%s", tlsTarget, utils.TypeDeref[string](tlsConfig.TLSCASecretRef.DataKey, defaultTLSCASecretKey)), + AutoTLS: false, + } + } + return "http", nil +} + +func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { + domainName := fmt.Sprintf("%s.%s.%s", etcd.GetPeerServiceName(), etcd.Namespace, "svc") + serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))) + builder := strings.Builder{} + for i := 0; i < int(etcd.Spec.Replicas); i++ { + podName := etcd.GetOrdinalPodName(i) + builder.WriteString(fmt.Sprintf("%s=%s://%s.%s:%s,", podName, peerScheme, podName, domainName, serverPort)) + } + return strings.Trim(builder.String(), ",") +} diff --git a/internal/resource/memberlease/memberlease.go b/internal/resource/memberlease/memberlease.go new file mode 100644 index 000000000..a53dcfb42 --- /dev/null +++ b/internal/resource/memberlease/memberlease.go @@ -0,0 +1,105 @@ +package memberlease + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/utils/flow" + "github.com/go-logr/logr" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const purpose = "etcd-member-lease" + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + leaseList := &coordinationv1.LeaseList{} + err := r.client.List(ctx, + leaseList, + client.InNamespace(etcd.Namespace), + client.MatchingLabels(getLabels(etcd))) + if err != nil { + return resourceNames, err + } + for _, lease := range leaseList.Items { + resourceNames = append(resourceNames, lease.Name) + } + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + objectKeys := getObjectKeys(etcd) + createFns := make([]flow.TaskFn, len(objectKeys)) + for _, objKey := range objectKeys { + objKey := objKey + createFns = append(createFns, func(ctx context.Context) error { + return r.doCreateOrUpdate(ctx, etcd, objKey) + }) + } + return flow.Parallel(createFns...)(ctx) +} + +func (r _resource) doCreateOrUpdate(ctx context.Context, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { + lease := emptyMemberLease(objKey) + opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { + lease.Labels = etcd.GetDefaultLabels() + lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + return nil + }) + if err != nil { + return err + } + r.logger.Info("triggered creation of member lease", "lease", objKey, "operationResult", opResult) + return nil +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return r.client.DeleteAllOf(ctx, + &coordinationv1.Lease{}, + client.InNamespace(etcd.Namespace), + client.MatchingLabels(etcd.GetDefaultLabels())) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { + leaseNames := etcd.GetMemberLeaseNames() + objectKeys := make([]client.ObjectKey, 0, len(leaseNames)) + for _, leaseName := range leaseNames { + objectKeys = append(objectKeys, client.ObjectKey{Name: leaseName, Namespace: etcd.Namespace}) + } + return objectKeys +} + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + labels := make(map[string]string) + labels[common.GardenerOwnedBy] = etcd.Name + labels[v1beta1constants.GardenerPurpose] = purpose + return utils.MergeMaps[string, string](labels, etcd.GetDefaultLabels()) +} + +func emptyMemberLease(objectKey client.ObjectKey) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} diff --git a/internal/resource/peerservice/peerservice.go b/internal/resource/peerservice/peerservice.go new file mode 100644 index 000000000..e325d8f11 --- /dev/null +++ b/internal/resource/peerservice/peerservice.go @@ -0,0 +1,95 @@ +package peerservice + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const defaultServerPort = 2380 + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + svc := &corev1.Service{} + if err := r.client.Get(ctx, getObjectKey(etcd), svc); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, svc.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + svc := emptyPeerService(getObjectKey(etcd)) + _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { + svc.Labels = etcd.GetDefaultLabels() + svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Spec.ClusterIP = corev1.ClusterIPNone + svc.Spec.SessionAffinity = corev1.ServiceAffinityNone + svc.Spec.Selector = etcd.GetDefaultLabels() + svc.Spec.PublishNotReadyAddresses = true + svc.Spec.Ports = getPorts(etcd) + return nil + }) + return err +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return client.IgnoreNotFound(r.client.Delete(ctx, emptyPeerService(getObjectKey(etcd)))) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace} +} + +func (r _resource) getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + var labelMaps []map[string]string + labelMaps = append(labelMaps, etcd.GetDefaultLabels()) + if etcd.Spec.Etcd.ClientService != nil { + labelMaps = append(labelMaps, etcd.Spec.Etcd.ClientService.Labels) + } + return utils.MergeMaps[string, string](labelMaps...) +} + +func emptyPeerService(objectKey client.ObjectKey) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} + +func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + return []corev1.ServicePort{ + { + Name: "peer", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }, + } +} diff --git a/internal/resource/poddistruptionbudget/poddisruptionbudget.go b/internal/resource/poddistruptionbudget/poddisruptionbudget.go new file mode 100644 index 000000000..11485abdb --- /dev/null +++ b/internal/resource/poddistruptionbudget/poddisruptionbudget.go @@ -0,0 +1,91 @@ +package poddistruptionbudget + +import ( + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + pdb := &policyv1.PodDisruptionBudget{} + if err := r.client.Get(ctx, getObjectKey(etcd), pdb); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, pdb.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + pdb := emptyPodDisruptionBudget(getObjectKey(etcd)) + _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { + pdb.Labels = etcd.GetDefaultLabels() + pdb.Annotations = getAnnotations(etcd) + pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + pdb.Spec.MinAvailable = &intstr.IntOrString{ + IntVal: computePDBMinAvailable(int(etcd.Spec.Replicas)), + Type: intstr.Int, + } + pdb.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: etcd.GetDefaultLabels(), + } + return nil + }) + return err +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return r.client.Delete(ctx, emptyPodDisruptionBudget(getObjectKey(etcd))) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace} +} + +func emptyPodDisruptionBudget(objectKey client.ObjectKey) *policyv1.PodDisruptionBudget { + return &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} + +func getAnnotations(etcd *druidv1alpha1.Etcd) map[string]string { + return map[string]string{ + common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), + common.GardenerOwnerType: "etcd", + } +} + +func computePDBMinAvailable(etcdReplicas int) int32 { + // do not enable PDB for single node cluster + if etcdReplicas <= 1 { + return 0 + } + return int32(etcdReplicas/2 + 1) +} diff --git a/internal/resource/resource.go b/internal/resource/resource.go new file mode 100644 index 000000000..91064fc43 --- /dev/null +++ b/internal/resource/resource.go @@ -0,0 +1,105 @@ +package resource + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/resource/clientservice" + "github.com/gardener/etcd-druid/internal/resource/configmap" + "github.com/gardener/etcd-druid/internal/resource/memberlease" + "github.com/gardener/etcd-druid/internal/resource/peerservice" + "github.com/gardener/etcd-druid/internal/resource/poddistruptionbudget" + "github.com/gardener/etcd-druid/internal/resource/role" + "github.com/gardener/etcd-druid/internal/resource/rolebinding" + "github.com/gardener/etcd-druid/internal/resource/serviceaccount" + "github.com/gardener/etcd-druid/internal/resource/snapshotlease" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ConfigMapCheckSumKey = "checksum/etcd-configmap" +) + +type OperatorContext struct { + context.Context + RunID string + Logger logr.Logger + Data map[string]string +} + +func NewOperatorContext(ctx context.Context, logger logr.Logger, runID string) OperatorContext { + return OperatorContext{ + Context: ctx, + RunID: runID, + Logger: logger, + Data: make(map[string]string), + } +} + +// Operator manages one or more resources of a specific Kind which are provisioned for an etcd cluster. +type Operator interface { + // GetExistingResourceNames gets all resources that currently exist that this Operator manages. + GetExistingResourceNames(ctx OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) + // TriggerDelete triggers the deletion of all resources that this Operator manages. + TriggerDelete(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error + // Sync synchronizes all resources that this Operator manages. If a resource does not exist then it will + // create it. If there are changes in the owning Etcd resource that transpires changes to one or more resources + // managed by this Operator then those resource(s) will be either be updated or a deletion is triggered. + Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error +} + +type OperatorConfig struct { + DisableEtcdServiceAccountAutomount bool +} + +type OperatorRegistry interface { + AllOperators() map[Kind]Operator + StatefulSetOperator() Operator + ServiceAccountOperator() Operator + RoleOperator() Operator + RoleBindingOperator() Operator + MemberLeaseOperator() Operator + SnapshotLeaseOperator() Operator + ConfigMapOperator() Operator + PeerServiceOperator() Operator + ClientServiceOperator() Operator + PodDisruptionBudgetOperator() Operator +} + +type Kind string + +const ( + StatefulSetKind Kind = "StatefulSet" + ServiceAccountKind Kind = "ServiceAccount" + RoleKind Kind = "Role" + RoleBindingKind Kind = "RoleBinding" + MemberLeaseKind Kind = "MemberLease" + SnapshotLeaseKind Kind = "SnapshotLease" + ConfigMapKind Kind = "ConfigMap" + PeerServiceKind Kind = "PeerService" + ClientServiceKind Kind = "ClientService" + PodDisruptionBudgetKind Kind = "PodDisruptionBudget" +) + +type registry struct { + operators map[Kind]Operator +} + +func NewOperatorRegistry(client client.Client, logger logr.Logger, config OperatorConfig) OperatorRegistry { + operators := make(map[Kind]Operator) + operators[ConfigMapKind] = configmap.New(client, logger) + operators[ServiceAccountKind] = serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount) + operators[MemberLeaseKind] = memberlease.New(client, logger) + operators[SnapshotLeaseKind] = snapshotlease.New(client, logger) + operators[ClientServiceKind] = clientservice.New(client, logger) + operators[PeerServiceKind] = peerservice.New(client, logger) + operators[PodDisruptionBudgetKind] = poddistruptionbudget.New(client, logger) + operators[RoleKind] = role.New(client, logger) + operators[RoleBindingKind] = rolebinding.New(client, logger) + return nil +} + +func (r registry) AllOperators() map[Kind]Operator { + return r.operators +} diff --git a/internal/resource/role/role.go b/internal/resource/role/role.go new file mode 100644 index 000000000..7cfe65cc8 --- /dev/null +++ b/internal/resource/role/role.go @@ -0,0 +1,85 @@ +package role + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + role := &rbacv1.Role{} + if err := r.client.Get(ctx, getObjectKey(etcd), role); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, role.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + role := emptyRole(etcd) + _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { + role.Labels = etcd.GetDefaultLabels() + role.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + role.Rules = createPolicyRules() + return nil + }) + return err +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return r.client.Delete(ctx, emptyRole(etcd)) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace} +} + +func emptyRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetRoleName(), + Namespace: etcd.Namespace, + }, + } +} + +func createPolicyRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"statefulsets"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + } +} diff --git a/internal/resource/rolebinding/rolebinding.go b/internal/resource/rolebinding/rolebinding.go new file mode 100644 index 000000000..4097b96ab --- /dev/null +++ b/internal/resource/rolebinding/rolebinding.go @@ -0,0 +1,88 @@ +package rolebinding + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + rb := &rbacv1.RoleBinding{} + if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, rb.Name) + return resourceNames, nil +} + +func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { + rb := &rbacv1.RoleBinding{} + if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + rb := emptyRoleBinding(etcd) + opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { + rb.Labels = etcd.GetDefaultLabels() + rb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + rb.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: etcd.GetRoleName(), + } + rb.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: etcd.GetServiceAccountName(), + Namespace: etcd.Namespace, + }, + } + return nil + }) + r.logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) + return err +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace} +} + +func emptyRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetRoleBindingName(), + Namespace: etcd.Namespace, + }, + } +} diff --git a/internal/resource/serviceaccount/serviceaccount.go b/internal/resource/serviceaccount/serviceaccount.go new file mode 100644 index 000000000..6c2d6cc70 --- /dev/null +++ b/internal/resource/serviceaccount/serviceaccount.go @@ -0,0 +1,70 @@ +package serviceaccount + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type _resource struct { + client client.Client + logger logr.Logger + etcd *druidv1alpha1.Etcd + disableAutoMount bool +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + sa := &corev1.ServiceAccount{} + if err := r.client.Get(ctx, getObjectKey(etcd), sa); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, sa.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + sa := emptyServiceAccount(getObjectKey(etcd)) + opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, sa, func() error { + sa.Labels = r.etcd.GetDefaultLabels() + sa.OwnerReferences = []metav1.OwnerReference{r.etcd.GetAsOwnerReference()} + sa.AutomountServiceAccountToken = pointer.Bool(r.disableAutoMount) + return nil + }) + r.logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) + return err +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return client.IgnoreNotFound(r.client.Delete(ctx, emptyServiceAccount(getObjectKey(etcd)))) +} + +func New(client client.Client, logger logr.Logger, disableAutomount bool) resource.Operator { + return &_resource{ + client: client, + logger: logger, + disableAutoMount: disableAutomount, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace} +} + +func emptyServiceAccount(objectKey client.ObjectKey) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} diff --git a/internal/resource/snapshotlease/snapshotlease.go b/internal/resource/snapshotlease/snapshotlease.go new file mode 100644 index 000000000..d3a4d7d39 --- /dev/null +++ b/internal/resource/snapshotlease/snapshotlease.go @@ -0,0 +1,128 @@ +package snapshotlease + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/go-logr/logr" + coordinationv1 "k8s.io/api/coordination/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const purpose = "etcd-snapshot-lease" + +type _resource struct { + client client.Client + logger logr.Logger +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 2) + // We have to get snapshot leases one lease at a time and cannot use label-selector based listing + // because currently snapshot lease do not have proper labels on them. In this new code + // we will add the labels. + // TODO: Once all snapshot leases have a purpose label on them, then we can use List instead of individual Get calls. + deltaSnapshotLease, err := r.getLease(ctx, + client.ObjectKey{Name: etcd.GetDeltaSnapshotLeaseName(), Namespace: etcd.Namespace}) + if err != nil { + return resourceNames, err + } + resourceNames = append(resourceNames, deltaSnapshotLease.Name) + fullSnapshotLease, err := r.getLease(ctx, + client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace}) + if err != nil { + return resourceNames, err + } + resourceNames = append(resourceNames, fullSnapshotLease.Name) + return resourceNames, nil +} + +func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*coordinationv1.Lease, error) { + lease := &coordinationv1.Lease{} + if err := r.client.Get(ctx, objectKey, lease); err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return lease, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + if !etcd.IsBackupEnabled() { + r.logger.Info("Backup has been disabled. Triggering delete of snapshot leases") + return r.TriggerDelete(ctx, etcd) + } + for _, objKey := range getObjectKeys(etcd) { + opResult, err := r.doSync(ctx, etcd, objKey) + if err != nil { + return err + } + r.logger.Info("Triggered create or update", "lease", objKey, "result", opResult) + } + return nil +} + +func (r _resource) doSync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) (controllerutil.OperationResult, error) { + lease := emptySnapshotLease(leaseObjectKey) + opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { + lease.Labels = getLabels(etcd) + lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + return nil + }) + return opResult, err +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + for _, objKey := range getObjectKeys(etcd) { + err := client.IgnoreNotFound(r.client.Delete(ctx, emptySnapshotLease(objKey))) + if err != nil { + return err + } + } + return nil +} + +func New(client client.Client, logger logr.Logger) resource.Operator { + return &_resource{ + client: client, + logger: logger, + } +} + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + labels := make(map[string]string) + labels[common.GardenerOwnedBy] = etcd.Name + labels[v1beta1constants.GardenerPurpose] = purpose + return utils.MergeMaps[string, string](labels, etcd.GetDefaultLabels()) +} + +func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { + return []client.ObjectKey{ + { + Name: etcd.GetDeltaSnapshotLeaseName(), + Namespace: etcd.Namespace, + }, + { + Name: etcd.GetFullSnapshotLeaseName(), + Namespace: etcd.Namespace, + }, + } +} + +func emptySnapshotLease(objectKey client.ObjectKey) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go new file mode 100644 index 000000000..aa53ac118 --- /dev/null +++ b/internal/utils/concurrent.go @@ -0,0 +1,77 @@ +package utils + +import ( + "fmt" + "runtime/debug" + "sync" + + "github.com/gardener/etcd-druid/internal/resource" +) + +// OperatorTask is a holder for a named function. +type OperatorTask struct { + // Name is the name of the task + Name string + // Fn is the function which accepts an operator context and returns an error if there is one. + // Implementations of Fn should handle context cancellation properly. + Fn func(ctx resource.OperatorContext) error +} + +// RunConcurrently runs tasks concurrently with number of goroutines bounded by bound. +// If there is a panic executing a single OperatorTask then it will capture the panic and capture it as an error +// which will then subsequently be returned from this function. It will not propagate the panic causing the app to exit. +func RunConcurrently(ctx resource.OperatorContext, tasks []OperatorTask) []error { + rg := newRunGroup(len(tasks)) + for _, task := range tasks { + rg.trigger(ctx, task) + } + return rg.WaitAndCollectErrors() +} + +// runGroup is a runner for concurrently spawning multiple asynchronous tasks. If any task +// errors or panics then these are captured as errors. +type runGroup struct { + wg sync.WaitGroup + errCh chan error +} + +// newRunGroup creates a new runGroup. +func newRunGroup(numTasks int) *runGroup { + return &runGroup{ + wg: sync.WaitGroup{}, + errCh: make(chan error, numTasks), + } +} + +// trigger executes the task in a go-routine. +func (g *runGroup) trigger(ctx resource.OperatorContext, task OperatorTask) { + g.wg.Add(1) + go func(task OperatorTask) { + defer g.wg.Done() + defer func() { + // recovers from a panic if there is one. Creates an error from it which contains the debug stack + // trace as well and pushes the error to the provided error channel. + if v := recover(); v != nil { + stack := debug.Stack() + panicErr := fmt.Errorf("task: %s execution panicked: %v\n, stack-trace: %s", task.Name, v, stack) + g.errCh <- panicErr + } + }() + err := task.Fn(ctx) + if err != nil { + g.errCh <- err + } + }(task) +} + +// WaitAndCollectErrors waits for all tasks to finish, collects and returns any errors. +func (g *runGroup) WaitAndCollectErrors() []error { + g.wg.Wait() + close(g.errCh) + + var errs []error + for err := range g.errCh { + errs = append(errs, err) + } + return errs +} diff --git a/internal/utils/image.go b/internal/utils/image.go new file mode 100755 index 000000000..8014c7bc7 --- /dev/null +++ b/internal/utils/image.go @@ -0,0 +1,83 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + "k8s.io/utils/pointer" +) + +func getEtcdImageKeys(useEtcdWrapper bool) (etcdImageKey string, etcdbrImageKey string, alpine string) { + alpine = common.Alpine + switch useEtcdWrapper { + case true: + etcdImageKey = common.EtcdWrapper + etcdbrImageKey = common.BackupRestoreDistroless + default: + etcdImageKey = common.Etcd + etcdbrImageKey = common.BackupRestore + } + return +} + +// GetEtcdImages returns images for etcd and backup-restore by inspecting the etcd spec and the image vector +// and returns the image for the init container by inspecting the image vector. +// It will give preference to images that are set in the etcd spec and only if the image is not found in it should +// it be picked up from the image vector if it's set there. +// A return value of nil for either of the images indicates that the image is not set. +func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcdWrapper bool) (*string, *string, *string, error) { + etcdImageKey, etcdbrImageKey, initContainerImageKey := getEtcdImageKeys(useEtcdWrapper) + etcdImage, err := chooseImage(etcdImageKey, etcd.Spec.Etcd.Image, iv) + if err != nil { + return nil, nil, nil, err + } + etcdBackupRestoreImage, err := chooseImage(etcdbrImageKey, etcd.Spec.Backup.Image, iv) + if err != nil { + return nil, nil, nil, err + } + initContainerImage, err := chooseImage(initContainerImageKey, nil, iv) + if err != nil { + return nil, nil, nil, err + } + + return etcdImage, etcdBackupRestoreImage, initContainerImage, nil +} + +// chooseImage selects an image based on the given key, specImage, and image vector. +// It returns the specImage if it is not nil; otherwise, it searches for the image in the image vector. +func chooseImage(key string, specImage *string, iv imagevector.ImageVector) (*string, error) { + if specImage != nil { + return specImage, nil + } + // Check if this image is present in the image vector. + ivImage, err := imagevector.FindImages(iv, []string{key}) + if err != nil { + return nil, err + } + return pointer.String(ivImage[key].String()), nil +} + +// GetEtcdBackupRestoreImage returns the image for backup-restore from the given image vector. +func GetEtcdBackupRestoreImage(iv imagevector.ImageVector, useEtcdWrapper bool) (*string, error) { + _, etcdbrImageKey, _ := getEtcdImageKeys(useEtcdWrapper) + return chooseImage(etcdbrImageKey, nil, iv) +} + +// GetInitContainerImage returns the image for init container from the given image vector. +func GetInitContainerImage(iv imagevector.ImageVector) (*string, error) { + return chooseImage(common.Alpine, nil, iv) +} diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go new file mode 100644 index 000000000..54734c493 --- /dev/null +++ b/internal/utils/image_test.go @@ -0,0 +1,172 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + testutils "github.com/gardener/etcd-druid/test/utils" + + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/pointer" +) + +var _ = Describe("Image retrieval tests", func() { + + const ( + etcdName = "etcd-test-0" + namespace = "default" + ) + var ( + imageVector imagevector.ImageVector + etcd *druidv1alpha1.Etcd + err error + ) + + It("etcd spec defines etcd and backup-restore images", func() { + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + imageVector = createImageVector(true, true, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) + Expect(err).To(BeNil()) + Expect(etcdImage).ToNot(BeNil()) + Expect(etcdImage).To(Equal(etcd.Spec.Etcd.Image)) + Expect(etcdBackupRestoreImage).ToNot(BeNil()) + Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) + vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) + Expect(err).To(BeNil()) + Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + }) + + It("etcd spec has no image defined and image vector has both images set", func() { + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + Expect(err).To(BeNil()) + etcd.Spec.Etcd.Image = nil + etcd.Spec.Backup.Image = nil + imageVector = createImageVector(true, true, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) + Expect(err).To(BeNil()) + Expect(etcdImage).ToNot(BeNil()) + vectorEtcdImage, err := imageVector.FindImage(common.Etcd) + Expect(err).To(BeNil()) + Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) + Expect(etcdBackupRestoreImage).ToNot(BeNil()) + vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestore) + Expect(err).To(BeNil()) + Expect(*etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) + vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) + Expect(err).To(BeNil()) + Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + }) + + It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + Expect(err).To(BeNil()) + etcd.Spec.Etcd.Image = nil + imageVector = createImageVector(true, false, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) + Expect(err).To(BeNil()) + Expect(etcdImage).ToNot(BeNil()) + vectorEtcdImage, err := imageVector.FindImage(common.Etcd) + Expect(err).To(BeNil()) + Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) + Expect(etcdBackupRestoreImage).ToNot(BeNil()) + Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) + vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) + Expect(err).To(BeNil()) + Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + }) + + It("both spec and image vector do not have backup-restore image", func() { + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + Expect(err).To(BeNil()) + etcd.Spec.Backup.Image = nil + imageVector = createImageVector(true, false, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) + Expect(err).ToNot(BeNil()) + Expect(etcdImage).To(BeNil()) + Expect(etcdBackupRestoreImage).To(BeNil()) + Expect(initContainerImage).To(BeNil()) + }) + + It("etcd spec has no images defined, image vector has all images, and UseEtcdWrapper feature gate is turned on", func() { + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + Expect(err).To(BeNil()) + etcd.Spec.Etcd.Image = nil + etcd.Spec.Backup.Image = nil + imageVector = createImageVector(true, true, true, true) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, true) + Expect(err).To(BeNil()) + Expect(etcdImage).ToNot(BeNil()) + vectorEtcdImage, err := imageVector.FindImage(common.EtcdWrapper) + Expect(err).To(BeNil()) + Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) + Expect(etcdBackupRestoreImage).ToNot(BeNil()) + vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestoreDistroless) + Expect(err).To(BeNil()) + Expect(*etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) + vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) + Expect(err).To(BeNil()) + Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + }) +}) + +func createImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperImage, withBackupRestoreDistrolessImage bool) imagevector.ImageVector { + var imageSources []*imagevector.ImageSource + const ( + repo = "test-repo" + etcdTag = "etcd-test-tag" + etcdWrapperTag = "etcd-wrapper-test-tag" + backupRestoreTag = "backup-restore-test-tag" + backupRestoreDistrolessTag = "backup-restore-distroless-test-tag" + initContainerTag = "init-container-test-tag" + ) + if withEtcdImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.Etcd, + Repository: repo, + Tag: pointer.String(etcdTag), + }) + } + if withBackupRestoreImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.BackupRestore, + Repository: repo, + Tag: pointer.String(backupRestoreTag), + }) + + } + if withEtcdWrapperImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.EtcdWrapper, + Repository: repo, + Tag: pointer.String(etcdWrapperTag), + }) + } + if withBackupRestoreDistrolessImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.BackupRestoreDistroless, + Repository: repo, + Tag: pointer.String(backupRestoreDistrolessTag), + }) + } + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.Alpine, + Repository: repo, + Tag: pointer.String(initContainerTag), + }) + return imageSources +} diff --git a/internal/utils/lease.go b/internal/utils/lease.go new file mode 100644 index 000000000..694baf5ec --- /dev/null +++ b/internal/utils/lease.go @@ -0,0 +1,74 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "strconv" + + "github.com/gardener/etcd-druid/pkg/common" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/go-logr/logr" + coordinationv1 "k8s.io/api/coordination/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// IsPeerURLTLSEnabled checks if the TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. +func IsPeerURLTLSEnabled(ctx context.Context, cli client.Client, namespace, etcdName string, logger logr.Logger) (bool, error) { + var tlsEnabledValues []bool + labels := GetMemberLeaseLabels(etcdName) + leaseList := &coordinationv1.LeaseList{} + if err := cli.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { + return false, err + } + for _, lease := range leaseList.Items { + tlsEnabled := parseAndGetTLSEnabledValue(lease, logger) + if tlsEnabled != nil { + tlsEnabledValues = append(tlsEnabledValues, *tlsEnabled) + } + } + tlsEnabled := true + for _, v := range tlsEnabledValues { + tlsEnabled = tlsEnabled && v + } + return tlsEnabled, nil +} + +// PurposeMemberLease is a constant used as a purpose for etcd member lease objects. +const PurposeMemberLease = "etcd-member-lease" + +// GetMemberLeaseLabels creates a map of default labels for member lease. +func GetMemberLeaseLabels(etcdName string) map[string]string { + return map[string]string{ + common.GardenerOwnedBy: etcdName, + v1beta1constants.GardenerPurpose: PurposeMemberLease, + } +} + +func parseAndGetTLSEnabledValue(lease coordinationv1.Lease, logger logr.Logger) *bool { + const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" + if lease.Annotations != nil { + if tlsEnabledStr, ok := lease.Annotations[peerURLTLSEnabledKey]; ok { + tlsEnabled, err := strconv.ParseBool(tlsEnabledStr) + if err != nil { + logger.Error(err, "tls-enabled value is not a valid boolean", "namespace", lease.Namespace, "leaseName", lease.Name) + return nil + } + return &tlsEnabled + } + logger.V(4).Info("tls-enabled annotation not present for lease.", "namespace", lease.Namespace, "leaseName", lease.Name) + } + return nil +} diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go new file mode 100644 index 000000000..3ab005a15 --- /dev/null +++ b/internal/utils/miscellaneous.go @@ -0,0 +1,102 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "maps" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MergeStringMaps merges the content of the newMaps with the oldMap. If a key already exists then +// it gets overwritten by the last value with the same key. +func MergeStringMaps(oldMap map[string]string, newMaps ...map[string]string) map[string]string { + var out map[string]string + + if oldMap != nil { + out = make(map[string]string) + } + for k, v := range oldMap { + out[k] = v + } + + for _, newMap := range newMaps { + if newMap != nil && out == nil { + out = make(map[string]string) + } + + for k, v := range newMap { + out[k] = v + } + } + + return out +} + +// MergeMaps merges the contents of maps. All maps will be processed in the order +// in which they are sent. For overlapping keys across source maps, value in the merged map +// for this key will be from the last occurrence of the key-value. +func MergeMaps[K comparable, V any](sourceMaps ...map[K]V) map[K]V { + if sourceMaps == nil { + return nil + } + merged := make(map[K]V) + for _, m := range sourceMaps { + maps.Copy(merged, m) + } + return merged +} + +func nameAndNamespace(namespaceOrName string, nameOpt ...string) (namespace, name string) { + if len(nameOpt) > 1 { + panic(fmt.Sprintf("more than name/namespace for key specified: %s/%v", namespaceOrName, nameOpt)) + } + if len(nameOpt) == 0 { + name = namespaceOrName + return + } + namespace = namespaceOrName + name = nameOpt[0] + return +} + +// Key creates a new client.ObjectKey from the given parameters. +// There are only two ways to call this function: +// - If only namespaceOrName is set, then a client.ObjectKey with name set to namespaceOrName is returned. +// - If namespaceOrName and one nameOpt is given, then a client.ObjectKey with namespace set to namespaceOrName +// and name set to nameOpt[0] is returned. +// +// For all other cases, this method panics. +func Key(namespaceOrName string, nameOpt ...string) client.ObjectKey { + namespace, name := nameAndNamespace(namespaceOrName, nameOpt...) + return client.ObjectKey{Namespace: namespace, Name: name} +} + +// Max returns the larger of x or y. +func Max(x, y int) int { + if y > x { + return y + } + return x +} + +// TypeDeref dereferences a pointer to a type if it is not nil, else it returns the default value. +func TypeDeref[T any](val *T, defaultVal T) T { + if val != nil { + return *val + } + return defaultVal +} diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go new file mode 100644 index 000000000..e07e09778 --- /dev/null +++ b/internal/utils/statefulset.go @@ -0,0 +1,63 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// IsStatefulSetReady checks whether the given StatefulSet is ready and up-to-date. +// A StatefulSet is considered healthy if its controller observed its current revision, +// it is not in an update (i.e. UpdateRevision is empty) and if its current replicas are equal to +// desired replicas specified in ETCD specs. +// It returns ready status (bool) and in case it is not ready then the second return value holds the reason. +func IsStatefulSetReady(etcdReplicas int32, statefulSet *appsv1.StatefulSet) (bool, string) { + if statefulSet.Status.ObservedGeneration < statefulSet.Generation { + return false, fmt.Sprintf("observed generation %d is outdated in comparison to generation %d", statefulSet.Status.ObservedGeneration, statefulSet.Generation) + } + if statefulSet.Status.ReadyReplicas < etcdReplicas { + return false, fmt.Sprintf("not enough ready replicas (%d/%d)", statefulSet.Status.ReadyReplicas, etcdReplicas) + } + if statefulSet.Status.CurrentRevision != statefulSet.Status.UpdateRevision { + return false, fmt.Sprintf("Current StatefulSet revision %s is older than the updated StatefulSet revision %s)", statefulSet.Status.CurrentRevision, statefulSet.Status.UpdateRevision) + } + if statefulSet.Status.CurrentReplicas != statefulSet.Status.UpdatedReplicas { + return false, fmt.Sprintf("StatefulSet status.CurrentReplicas (%d) != status.UpdatedReplicas (%d)", statefulSet.Status.CurrentReplicas, statefulSet.Status.UpdatedReplicas) + } + return true, "" +} + +// GetStatefulSet fetches StatefulSet created for the etcd. +func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { + statefulSets := &appsv1.StatefulSetList{} + if err := cl.List(ctx, statefulSets, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: labels.Set(etcd.GetDefaultLabels()).AsSelector()}); err != nil { + return nil, err + } + + for _, sts := range statefulSets.Items { + if metav1.IsControlledBy(&sts, etcd) { + return &sts, nil + } + } + + return nil, nil +} diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go new file mode 100644 index 000000000..dd5d2a70b --- /dev/null +++ b/internal/utils/statefulset_test.go @@ -0,0 +1,121 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + "github.com/gardener/etcd-druid/test/utils" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("tests for statefulset utility functions", func() { + + var ( + ctx context.Context + fakeClient = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + ) + + Describe("#IsStatefulSetReady", func() { + const ( + stsName = "etcd-test-0" + stsNamespace = "test-ns" + ) + It("statefulset has less number of ready replicas as compared to configured etcd replicas", func() { + sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 2) + sts.Generation = 1 + sts.Status.ObservedGeneration = 1 + sts.Status.Replicas = 2 + sts.Status.ReadyReplicas = 2 + ready, reasonMsg := IsStatefulSetReady(3, sts) + Expect(ready).To(BeFalse()) + Expect(reasonMsg).ToNot(BeNil()) + }) + + It("statefulset has equal number of replicas as defined in etcd but observed generation is outdated", func() { + sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) + sts.Generation = 2 + sts.Status.ObservedGeneration = 1 + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + ready, reasonMsg := IsStatefulSetReady(3, sts) + Expect(ready).To(BeFalse()) + Expect(reasonMsg).ToNot(BeNil()) + }) + + It("statefulset has equal number of replicas as defined in etcd and observed generation = generation", func() { + sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) + utils.SetStatefulSetReady(sts) + ready, reasonMsg := IsStatefulSetReady(3, sts) + Expect(ready).To(BeTrue()) + Expect(len(reasonMsg)).To(BeZero()) + }) + }) + + Describe("#GetStatefulSet", func() { + var ( + etcd *druidv1alpha1.Etcd + stsListToCleanup *appsv1.StatefulSetList + ) + const ( + testEtcdName = "etcd-test-0" + testNamespace = "test-ns" + ) + + BeforeEach(func() { + ctx = context.TODO() + stsListToCleanup = &appsv1.StatefulSetList{} + etcd = utils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() + }) + + AfterEach(func() { + for _, sts := range stsListToCleanup.Items { + Expect(fakeClient.Delete(ctx, &sts)).To(Succeed()) + } + }) + + It("no statefulset is found irrespective of ownership", func() { + sts, err := GetStatefulSet(ctx, fakeClient, etcd) + Expect(err).To(BeNil()) + Expect(sts).To(BeNil()) + }) + + It("statefulset is present but it is not owned by etcd", func() { + sts := utils.CreateStatefulSet(etcd.Name, etcd.Namespace, uuid.NewUUID(), 3) + Expect(fakeClient.Create(ctx, sts)).To(Succeed()) + stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) + foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) + Expect(err).To(BeNil()) + Expect(foundSts).To(BeNil()) + }) + + It("found statefulset owned by etcd", func() { + sts := utils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, 3) + Expect(fakeClient.Create(ctx, sts)).To(Succeed()) + stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) + foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) + Expect(err).To(BeNil()) + Expect(foundSts).ToNot(BeNil()) + Expect(foundSts.UID).To(Equal(sts.UID)) + }) + }) +}) diff --git a/internal/utils/store.go b/internal/utils/store.go new file mode 100644 index 000000000..4334041d3 --- /dev/null +++ b/internal/utils/store.go @@ -0,0 +1,110 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "fmt" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // LocalProviderDefaultMountPath is the default path where the buckets directory is mounted. + LocalProviderDefaultMountPath = "/etc/gardener/local-backupbuckets" + // EtcdBackupSecretHostPath is the hostPath field in the etcd-backup secret. + EtcdBackupSecretHostPath = "hostPath" +) + +const ( + aws = "aws" + azure = "azure" + gcp = "gcp" + alicloud = "alicloud" + openstack = "openstack" + dell = "dell" + openshift = "openshift" +) + +const ( + // S3 is a constant for the AWS and S3 compliant storage provider. + S3 = "S3" + // ABS is a constant for the Azure storage provider. + ABS = "ABS" + // GCS is a constant for the Google storage provider. + GCS = "GCS" + // OSS is a constant for the Alicloud storage provider. + OSS = "OSS" + // Swift is a constant for the OpenStack storage provider. + Swift = "Swift" + // Local is a constant for the Local storage provider. + Local = "Local" + // ECS is a constant for the EMC storage provider. + ECS = "ECS" + // OCS is a constant for the OpenShift storage provider. + OCS = "OCS" +) + +// GetHostMountPathFromSecretRef returns the hostPath configured for the given store. +func GetHostMountPathFromSecretRef(ctx context.Context, client client.Client, logger logr.Logger, store *druidv1alpha1.StoreSpec, namespace string) (string, error) { + if store.SecretRef == nil { + logger.Info("secretRef is not defined for store, using default hostPath", "namespace", namespace) + return LocalProviderDefaultMountPath, nil + } + + secret := &corev1.Secret{} + if err := client.Get(ctx, Key(namespace, store.SecretRef.Name), secret); err != nil { + return "", err + } + + hostPath, ok := secret.Data[EtcdBackupSecretHostPath] + if !ok { + return LocalProviderDefaultMountPath, nil + } + + return string(hostPath), nil +} + +// StorageProviderFromInfraProvider converts infra to object store provider. +func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (string, error) { + if infra == nil || len(*infra) == 0 { + return "", nil + } + + switch *infra { + case aws, S3: + return S3, nil + case azure, ABS: + return ABS, nil + case alicloud, OSS: + return OSS, nil + case openstack, Swift: + return Swift, nil + case gcp, GCS: + return GCS, nil + case dell, ECS: + return ECS, nil + case openshift, OCS: + return OCS, nil + case Local, druidv1alpha1.StorageProvider(strings.ToLower(Local)): + return Local, nil + default: + return "", fmt.Errorf("unsupported storage provider: %v", *infra) + } +} diff --git a/internal/utils/store_test.go b/internal/utils/store_test.go new file mode 100644 index 000000000..966980aa1 --- /dev/null +++ b/internal/utils/store_test.go @@ -0,0 +1,118 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/utils/test/matchers" + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + corev1 "k8s.io/api/core/v1" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Store tests", func() { + + Describe("#GetHostMountPathFromSecretRef", func() { + var ( + ctx context.Context + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + storeSpec *druidv1alpha1.StoreSpec + sec *corev1.Secret + logger = logr.Discard() + ) + const ( + testNamespace = "test-ns" + ) + + BeforeEach(func() { + ctx = context.Background() + storeSpec = &druidv1alpha1.StoreSpec{} + }) + + AfterEach(func() { + if sec != nil { + err := fakeClient.Delete(ctx, sec) + if err != nil { + Expect(err).To(matchers.BeNotFoundError()) + } + } + }) + + It("no secret ref configured, should return default mount path", func() { + hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(hostMountPath).To(Equal(LocalProviderDefaultMountPath)) + }) + + It("secret ref points to an unknown secret, should return an error", func() { + storeSpec.SecretRef = &corev1.SecretReference{ + Name: "not-to-be-found-secret-ref", + Namespace: testNamespace, + } + _, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) + Expect(err).ToNot(BeNil()) + Expect(err).To(matchers.BeNotFoundError()) + }) + + It("secret ref points to a secret whose data does not have path set, should return default mount path", func() { + const secretName = "backup-secret" + sec = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: testNamespace, + }, + Data: map[string][]byte{"bucketName": []byte("NDQ5YjEwZj")}, + } + Expect(fakeClient.Create(ctx, sec)).To(Succeed()) + storeSpec.SecretRef = &corev1.SecretReference{ + Name: "backup-secret", + Namespace: testNamespace, + } + hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(hostMountPath).To(Equal(LocalProviderDefaultMountPath)) + }) + + It("secret ref points to a secret whose data has a path, should return the path defined in secret.Data", func() { + const ( + secretName = "backup-secret" + hostPath = "/var/data/etcd-backup" + ) + sec = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: testNamespace, + }, + Data: map[string][]byte{"bucketName": []byte("NDQ5YjEwZj"), "hostPath": []byte(hostPath)}, + } + Expect(fakeClient.Create(ctx, sec)).To(Succeed()) + storeSpec.SecretRef = &corev1.SecretReference{ + Name: "backup-secret", + Namespace: testNamespace, + } + hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(hostMountPath).To(Equal(hostPath)) + }) + }) +}) diff --git a/internal/utils/utils_suite_test.go b/internal/utils/utils_suite_test.go new file mode 100644 index 000000000..9f242798f --- /dev/null +++ b/internal/utils/utils_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Utils Suite") +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 000000000..d13d533ec --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,24 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +var ( + // These variables are typically populated using -ldflags settings when building the binary + + // Version stores the etcd-druid binary version. + Version string + // GitSHA stores the etcd-druid binary code commit SHA on git. + GitSHA string +) From 82f40a8557c1862497dd398c5a33e6b1b7593a26 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sun, 19 Nov 2023 23:23:08 +0530 Subject: [PATCH 002/235] Adapt custodian controller to a recovery-controller --- internal/controller/config.go | 2 +- internal/controller/custodian/config.go | 44 ++++++++ internal/controller/custodian/reconciler.go | 89 ++++++++++++++++ internal/controller/custodian/register.go | 61 +++++++++++ internal/controller/etcd/register.go | 7 +- internal/controller/manager.go | 8 +- internal/controller/predicate/predicate.go | 111 ++++++++++++++------ 7 files changed, 281 insertions(+), 41 deletions(-) create mode 100644 internal/controller/custodian/config.go create mode 100644 internal/controller/custodian/reconciler.go create mode 100644 internal/controller/custodian/register.go diff --git a/internal/controller/config.go b/internal/controller/config.go index b53dec2c6..0e165535e 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -17,8 +17,8 @@ package controller import ( "fmt" - "github.com/gardener/etcd-druid/controllers/custodian" "github.com/gardener/etcd-druid/internal/controller/compaction" + "github.com/gardener/etcd-druid/internal/controller/custodian" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" diff --git a/internal/controller/custodian/config.go b/internal/controller/custodian/config.go new file mode 100644 index 000000000..0fa322326 --- /dev/null +++ b/internal/controller/custodian/config.go @@ -0,0 +1,44 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package custodian + +import ( + "github.com/gardener/etcd-druid/controllers/utils" + + flag "github.com/spf13/pflag" +) + +const ( + workersFlagName = "custodian-workers" + + defaultCustodianWorkers = 3 +) + +// Config contains configuration for the Custodian Controller. +type Config struct { + // Workers denotes the number of worker threads for the custodian controller. + Workers int +} + +// InitFromFlags initializes the config from the provided CLI flag set. +func InitFromFlags(fs *flag.FlagSet, cfg *Config) { + fs.IntVar(&cfg.Workers, workersFlagName, defaultCustodianWorkers, + "Number of worker threads for the custodian controller.") +} + +// Validate validates the config. +func (cfg *Config) Validate() error { + return utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers) +} diff --git a/internal/controller/custodian/reconciler.go b/internal/controller/custodian/reconciler.go new file mode 100644 index 000000000..04eca8230 --- /dev/null +++ b/internal/controller/custodian/reconciler.go @@ -0,0 +1,89 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package custodian + +import ( + "context" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// Reconciler reconciles status of Etcd object +type Reconciler struct { + client.Client + scheme *runtime.Scheme + config *Config + logger logr.Logger +} + +// NewReconciler creates a new reconciler for Custodian. +func NewReconciler(mgr manager.Manager, config *Config) *Reconciler { + return &Reconciler{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + config: config, + logger: log.Log.WithName(controllerName), + } +} + +// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;patch +// +kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;watch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch +// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch + +// Reconcile reconciles the etcd. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.logger.Info("Custodian controller reconciliation started") + etcd := &druidv1alpha1.Etcd{} + if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) + + if !metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) && + etcd.Status.LastOperation != nil && etcd.Status.LastOperation.State != druidv1alpha1.LastOperationStateProcessing { + if err := r.triggerEtcdReconcile(ctx, logger, etcd); err != nil { + return ctrl.Result{Requeue: true}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *Reconciler) triggerEtcdReconcile(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { + logger.Info("Adding operation annotation", "annotation", v1beta1constants.GardenerOperation) + withOpAnnotation := etcd.DeepCopy() + withOpAnnotation.Annotations[v1beta1constants.GardenerOperation] = v1beta1constants.GardenerOperationReconcile + return r.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)) +} diff --git a/internal/controller/custodian/register.go b/internal/controller/custodian/register.go new file mode 100644 index 000000000..31be80bb2 --- /dev/null +++ b/internal/controller/custodian/register.go @@ -0,0 +1,61 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package custodian + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druidpredicates "github.com/gardener/etcd-druid/internal/controller/predicate" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlbuilder "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +const controllerName = "custodian-controller" + +// RegisterWithManager registers the Custodian Controller with the given controller manager. +func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { + return ctrl. + NewControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.config.Workers, + }). + For(&druidv1alpha1.Etcd{}). + Owns(&corev1.Service{}). + Owns(&corev1.ConfigMap{}). + Owns(&coordinationv1.Lease{}). + Owns(&policyv1.PodDisruptionBudget{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&appsv1.StatefulSet{}, + // ignore (VPA) updates to statefulset container resources + ctrlbuilder.WithPredicates( + predicate.And( + druidpredicates.StatefulSetSpecChange(), + predicate.Not( + druidpredicates.StatefulSetContainerResourcesChange(), + ), + ), + ), + ). + Complete(r) +} diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 27fae2f24..ddfbb97a5 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -18,14 +18,9 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { Named(controllerName). WithOptions(controller.Options{ MaxConcurrentReconciles: r.config.Workers, - // TODO: check if necessary - RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(10*time.Millisecond, 2*r.config.EtcdStatusSyncPeriod), + RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(10*time.Millisecond, r.config.EtcdStatusSyncPeriod), }). For(&druidv1alpha1.Etcd{}) return builder.Complete(r) } - -// TODO: create new etcd-recovery-controller which Owns (watches) all created resources -// If any of the owned resources is deleted/updated, and ignore-reconciliation annotation is not present on the etcd resource, -// then add the gardener.cloud/operation=reconcile on the etcd (if IgnoreOperationAnnotation is set to false) diff --git a/internal/controller/manager.go b/internal/controller/manager.go index c6d9447cd..70a29d8eb 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -18,9 +18,9 @@ import ( "context" "time" - "github.com/gardener/etcd-druid/controllers/custodian" "github.com/gardener/etcd-druid/internal/client/kubernetes" "github.com/gardener/etcd-druid/internal/controller/compaction" + "github.com/gardener/etcd-druid/internal/controller/custodian" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" @@ -96,9 +96,7 @@ func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) err if err != nil { return err } - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) - defer cancel() - if err = custodianReconciler.RegisterWithManager(ctx, mgr, config.IgnoreOperationAnnotation); err != nil { + if err = custodianReconciler.RegisterWithManager(mgr); err != nil { return err } @@ -123,6 +121,8 @@ func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) err } // Add secret reconciler to the manager + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() return secret.NewReconciler( mgr, config.SecretControllerConfig, diff --git a/internal/controller/predicate/predicate.go b/internal/controller/predicate/predicate.go index 260f69587..ba588f1b3 100644 --- a/internal/controller/predicate/predicate.go +++ b/internal/controller/predicate/predicate.go @@ -22,6 +22,7 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -82,36 +83,6 @@ func LastOperationNotSuccessful() predicate.Predicate { } } -// StatefulSetStatusChange is a predicate for status changes of `StatefulSet` resources. -func StatefulSetStatusChange() predicate.Predicate { - statusChange := func(objOld, objNew client.Object) bool { - stsOld, ok := objOld.(*appsv1.StatefulSet) - if !ok { - return false - } - stsNew, ok := objNew.(*appsv1.StatefulSet) - if !ok { - return false - } - return !apiequality.Semantic.DeepEqual(stsOld.Status, stsNew.Status) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return true - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return statusChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return true - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return true - }, - } -} - // EtcdReconciliationFinished is a predicate to use for etcd resources whose reconciliation has finished. func EtcdReconciliationFinished(ignoreOperationAnnotation bool) predicate.Predicate { reconciliationFinished := func(obj client.Object) bool { @@ -219,3 +190,83 @@ func JobStatusChanged() predicate.Predicate { }, } } + +// StatefulSetSpecChange is a predicate for status changes of `StatefulSet` resources. +func StatefulSetSpecChange() predicate.Predicate { + specChange := func(objOld, objNew client.Object) bool { + stsOld, ok := objOld.(*appsv1.StatefulSet) + if !ok { + return false + } + stsNew, ok := objNew.(*appsv1.StatefulSet) + if !ok { + return false + } + return !apiequality.Semantic.DeepEqual(stsOld.Spec, stsNew.Spec) + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return true + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return specChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return true + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return true + }, + } +} + +func StatefulSetContainerResourcesChange() predicate.Predicate { + containerResourcesChange := func(objOld, objNew client.Object) bool { + stsOld, ok := objOld.(*appsv1.StatefulSet) + if !ok { + return false + } + stsNew, ok := objNew.(*appsv1.StatefulSet) + if !ok { + return false + } + + containersOld := make(map[string]*corev1.Container) + containersNew := make(map[string]*corev1.Container) + + if len(stsOld.Spec.Template.Spec.Containers) != len(stsNew.Spec.Template.Spec.Containers) { + return true + } + for _, c := range stsOld.Spec.Template.Spec.Containers { + containersOld[c.Name] = &c + } + for _, cNew := range stsNew.Spec.Template.Spec.Containers { + containersNew[cNew.Name] = &cNew + cOld, ok := containersOld[cNew.Name] + if !ok { + return true + } + if !apiequality.Semantic.DeepEqual(cNew.Resources, cOld.Resources) { + return true + } + } + + return false + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return false + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return containerResourcesChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return false + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return false + }, + } +} From 1d0e8a88d226cf2c45dd33de465796448712a8b7 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 20 Nov 2023 10:51:05 +0530 Subject: [PATCH 003/235] Add missing JSON tags to fields in types_etcd.go --- api/v1alpha1/types_etcd.go | 12 ++-- api/v1alpha1/zz_generated.deepcopy.go | 44 +++++++++++++ .../10-crd-druid.gardener.cloud_etcds.yaml | 61 ++++++++++++++++++- .../10-crd-druid.gardener.cloud_etcds.yaml | 61 ++++++++++++++++++- 4 files changed, 170 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index e3c8b99fb..da3bd3837 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -387,9 +387,9 @@ type EtcdStatus struct { LastError *string `json:"lastError,omitempty"` // LastErrors captures errors that occurred during the last operation. // +optional - LastErrors []LastError + LastErrors []LastError `json:"lastErrors,omitempty"` // LastOperation indicates the last operation performed on this resource. - LastOperation *LastOperation + LastOperation *LastOperation `json:"lastOperation,omitempty"` // Cluster size is the current size of the etcd cluster. // Deprecated: this field will not be populated with any value and will be removed in the future. // +optional @@ -460,7 +460,7 @@ type LastOperation struct { // generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this // as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering // reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. - RunID string + RunID string `json:"rundID"` // LastUpdateTime is the time at which the operation was updated. LastUpdateTime metav1.Time `json:"lastUpdateTime"` } @@ -470,11 +470,11 @@ type ErrorCode string type LastError struct { // Code is an error code that uniquely identifies an error. - Code ErrorCode + Code ErrorCode `json:"code"` // Description is a human-readable message indicating details of the error. - Description string + Description string `json:"description"` // LastUpdateTime is the time the error was reported. - LastUpdateTime metav1.Time + LastUpdateTime metav1.Time `json:"lastUpdateTime"` } // GetNamespaceName is a convenience function which creates a types.NamespacedName for an etcd resource. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d7f4f903e..ab4a188ba 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -576,6 +576,18 @@ func (in *EtcdStatus) DeepCopyInto(out *EtcdStatus) { *out = new(string) **out = **in } + if in.LastErrors != nil { + in, out := &in.LastErrors, &out.LastErrors + *out = make([]LastError, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.LastOperation != nil { + in, out := &in.LastOperation, &out.LastOperation + *out = new(LastOperation) + (*in).DeepCopyInto(*out) + } if in.ClusterSize != nil { in, out := &in.ClusterSize, &out.ClusterSize *out = new(int32) @@ -615,6 +627,38 @@ func (in *EtcdStatus) DeepCopy() *EtcdStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LastError) DeepCopyInto(out *LastError) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LastError. +func (in *LastError) DeepCopy() *LastError { + if in == nil { + return nil + } + out := new(LastError) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LastOperation) DeepCopyInto(out *LastOperation) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LastOperation. +func (in *LastOperation) DeepCopy() *LastOperation { + if in == nil { + return nil + } + out := new(LastOperation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LeaderElectionSpec) DeepCopyInto(out *LeaderElectionSpec) { *out = *in diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index 3048503a6..cb75fb2a9 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -1834,8 +1834,67 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: LastError represents the last occurred error. + description: 'LastError represents the last occurred error. Deprecated: + Use LastErrors instead.' type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + lastUpdateTime: + description: LastUpdateTime is the time the error was reported. + format: date-time + type: string + required: + - code + - description + - lastUpdateTime + type: object + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. + properties: + description: + description: Description describes the last operation. + type: string + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was updated. + format: date-time + type: string + rundID: + description: RunID correlates an operation with a reconciliation + run. Every time an etcd resource is reconciled (barring status + reconciliation which is periodic), a unique ID is generated + which can be used to correlate all actions done as part of a + single reconcile run. Capturing this as part of LastOperation + aids in establishing this correlation. This further helps in + also easily filtering reconcile logs as all structured logs + in a reconcile run should have the `runID` referenced. + type: string + state: + description: State is the state of the last operation. + type: string + type: + description: Type is the type of last operation. + type: string + required: + - description + - lastUpdateTime + - rundID + - state + - type + type: object members: description: Members represents the members of the etcd cluster items: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 3048503a6..cb75fb2a9 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -1834,8 +1834,67 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: LastError represents the last occurred error. + description: 'LastError represents the last occurred error. Deprecated: + Use LastErrors instead.' type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + lastUpdateTime: + description: LastUpdateTime is the time the error was reported. + format: date-time + type: string + required: + - code + - description + - lastUpdateTime + type: object + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. + properties: + description: + description: Description describes the last operation. + type: string + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was updated. + format: date-time + type: string + rundID: + description: RunID correlates an operation with a reconciliation + run. Every time an etcd resource is reconciled (barring status + reconciliation which is periodic), a unique ID is generated + which can be used to correlate all actions done as part of a + single reconcile run. Capturing this as part of LastOperation + aids in establishing this correlation. This further helps in + also easily filtering reconcile logs as all structured logs + in a reconcile run should have the `runID` referenced. + type: string + state: + description: State is the state of the last operation. + type: string + type: + description: Type is the type of last operation. + type: string + required: + - description + - lastUpdateTime + - rundID + - state + - type + type: object members: description: Members represents the members of the etcd cluster items: From 052a1cbf3e47e1cb84963524b04260adf290570c Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 20 Nov 2023 10:56:50 +0530 Subject: [PATCH 004/235] Fixed error by Updating Method Name to GetConfigMapName --- api/v1alpha1/types_etcd_test.go | 2 +- pkg/component/etcd/configmap/values_helper.go | 2 +- pkg/component/etcd/statefulset/values_helper.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 4c758b545..47a468f47 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -77,7 +77,7 @@ var _ = Describe("Etcd", func() { Context("GetConfigmapName", func() { It("should return the correct configmap name", func() { - Expect(created.GetConfigmapName()).To(Equal("etcd-bootstrap-123456")) + Expect(created.GetConfigMapName()).To(Equal("etcd-bootstrap-123456")) }) }) diff --git a/pkg/component/etcd/configmap/values_helper.go b/pkg/component/etcd/configmap/values_helper.go index fa5445704..2929e1cef 100644 --- a/pkg/component/etcd/configmap/values_helper.go +++ b/pkg/component/etcd/configmap/values_helper.go @@ -17,7 +17,7 @@ import ( func GenerateValues(etcd *druidv1alpha1.Etcd) *Values { initialCluster := prepareInitialCluster(etcd) values := &Values{ - Name: etcd.GetConfigmapName(), + Name: etcd.GetConfigMapName(), EtcdUID: etcd.UID, Metrics: etcd.Spec.Etcd.Metrics, Quota: etcd.Spec.Etcd.Quota, diff --git a/pkg/component/etcd/statefulset/values_helper.go b/pkg/component/etcd/statefulset/values_helper.go index c600b3c80..d7bf0ec29 100644 --- a/pkg/component/etcd/statefulset/values_helper.go +++ b/pkg/component/etcd/statefulset/values_helper.go @@ -111,7 +111,7 @@ func GenerateValues( AutoCompactionMode: etcd.Spec.Common.AutoCompactionMode, AutoCompactionRetention: etcd.Spec.Common.AutoCompactionRetention, - ConfigMapName: etcd.GetConfigmapName(), + ConfigMapName: etcd.GetConfigMapName(), PeerTLSChangedToEnabled: peerTLSChangedToEnabled, UseEtcdWrapper: useEtcdWrapper, From 63cd1ba30bc585b0c1589b184c28768bb14936ca Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 20 Nov 2023 11:25:25 +0530 Subject: [PATCH 005/235] added missing annotation constants --- api/v1alpha1/types_common.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/v1alpha1/types_common.go b/api/v1alpha1/types_common.go index 301bc1090..5b601eeb7 100644 --- a/api/v1alpha1/types_common.go +++ b/api/v1alpha1/types_common.go @@ -63,3 +63,11 @@ type Condition struct { // A human readable message indicating details about the transition. Message string `json:"message"` } + +const ( + // IgnoreReconciliationAnnotation is an annotation set by an operator in order to stop reconciliation. + // Deprecated: Please use SuspendEtcdSpecReconcileAnnotation instead + IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" + // SuspendEtcdSpecReconcileAnnotation is an annotation set by an operator to temporarily suspend any etcd spec reconciliation. + SuspendEtcdSpecReconcileAnnotation = "druid.gardener.cloud/suspend-etcd-spec-reconcile" +) From 268defe722ebed78226aff4c0ff0721bc46f6476 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 20 Nov 2023 14:09:42 +0530 Subject: [PATCH 006/235] Refactor resource pkg to resolve Cyclic Imports Introduced a new `registry` package containing `OperatorRegistry`. Additionally, created a subpackage `registry/resource` contains OperatorContext and Operator interface to address cyclic import issues --- internal/controller/etcd/reconcile_delete.go | 2 +- internal/controller/etcd/reconciler.go | 9 +-- internal/controller/utils/etcdstatus.go | 2 +- .../resource.go => registry/registry.go} | 67 +++++-------------- internal/registry/resource/resource.go | 40 +++++++++++ .../resource/clientservice/clientservice.go | 2 +- internal/resource/configmap/configmap.go | 2 +- internal/resource/memberlease/memberlease.go | 3 +- internal/resource/peerservice/peerservice.go | 2 +- .../poddisruptionbudget.go | 2 +- internal/resource/role/role.go | 2 +- internal/resource/rolebinding/rolebinding.go | 2 +- .../resource/serviceaccount/serviceaccount.go | 2 +- .../resource/snapshotlease/snapshotlease.go | 2 +- internal/utils/concurrent.go | 2 +- 15 files changed, 75 insertions(+), 66 deletions(-) rename internal/{resource/resource.go => registry/registry.go} (53%) create mode 100644 internal/registry/resource/resource.go diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 3cd1c7005..fd2e0b136 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -7,7 +7,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 934ab7fe2..ff67cff61 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -5,7 +5,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" "github.com/google/uuid" @@ -23,7 +24,7 @@ type Reconciler struct { config *Config recorder record.EventRecorder imageVector imagevector.ImageVector - operatorRegistry resource.OperatorRegistry + operatorRegistry registry.OperatorRegistry lastOpErrRecorder ctrlutils.LastOperationErrorRecorder logger logr.Logger } @@ -35,9 +36,9 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { if err != nil { return nil, err } - operatorReg := resource.NewOperatorRegistry(mgr.GetClient(), + operatorReg := registry.NewOperatorRegistry(mgr.GetClient(), logger, - resource.OperatorConfig{ + registry.OperatorConfig{ DisableEtcdServiceAccountAutomount: config.DisableEtcdServiceAccountAutomount, }, ) diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index fc2292774..437cba2b9 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/resource/resource.go b/internal/registry/registry.go similarity index 53% rename from internal/resource/resource.go rename to internal/registry/registry.go index 91064fc43..b1d3d3657 100644 --- a/internal/resource/resource.go +++ b/internal/registry/registry.go @@ -1,9 +1,7 @@ -package resource +package registry import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/etcd-druid/internal/resource/clientservice" "github.com/gardener/etcd-druid/internal/resource/configmap" "github.com/gardener/etcd-druid/internal/resource/memberlease" @@ -13,58 +11,27 @@ import ( "github.com/gardener/etcd-druid/internal/resource/rolebinding" "github.com/gardener/etcd-druid/internal/resource/serviceaccount" "github.com/gardener/etcd-druid/internal/resource/snapshotlease" + "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - ConfigMapCheckSumKey = "checksum/etcd-configmap" -) - -type OperatorContext struct { - context.Context - RunID string - Logger logr.Logger - Data map[string]string -} - -func NewOperatorContext(ctx context.Context, logger logr.Logger, runID string) OperatorContext { - return OperatorContext{ - Context: ctx, - RunID: runID, - Logger: logger, - Data: make(map[string]string), - } -} - -// Operator manages one or more resources of a specific Kind which are provisioned for an etcd cluster. -type Operator interface { - // GetExistingResourceNames gets all resources that currently exist that this Operator manages. - GetExistingResourceNames(ctx OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) - // TriggerDelete triggers the deletion of all resources that this Operator manages. - TriggerDelete(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error - // Sync synchronizes all resources that this Operator manages. If a resource does not exist then it will - // create it. If there are changes in the owning Etcd resource that transpires changes to one or more resources - // managed by this Operator then those resource(s) will be either be updated or a deletion is triggered. - Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error -} - type OperatorConfig struct { DisableEtcdServiceAccountAutomount bool } type OperatorRegistry interface { - AllOperators() map[Kind]Operator - StatefulSetOperator() Operator - ServiceAccountOperator() Operator - RoleOperator() Operator - RoleBindingOperator() Operator - MemberLeaseOperator() Operator - SnapshotLeaseOperator() Operator - ConfigMapOperator() Operator - PeerServiceOperator() Operator - ClientServiceOperator() Operator - PodDisruptionBudgetOperator() Operator + AllOperators() map[Kind]resource.Operator + StatefulSetOperator() resource.Operator + ServiceAccountOperator() resource.Operator + RoleOperator() resource.Operator + RoleBindingOperator() resource.Operator + MemberLeaseOperator() resource.Operator + SnapshotLeaseOperator() resource.Operator + ConfigMapOperator() resource.Operator + PeerServiceOperator() resource.Operator + ClientServiceOperator() resource.Operator + PodDisruptionBudgetOperator() resource.Operator } type Kind string @@ -83,11 +50,11 @@ const ( ) type registry struct { - operators map[Kind]Operator + operators map[Kind]resource.Operator } func NewOperatorRegistry(client client.Client, logger logr.Logger, config OperatorConfig) OperatorRegistry { - operators := make(map[Kind]Operator) + operators := make(map[Kind]resource.Operator) operators[ConfigMapKind] = configmap.New(client, logger) operators[ServiceAccountKind] = serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount) operators[MemberLeaseKind] = memberlease.New(client, logger) @@ -100,6 +67,6 @@ func NewOperatorRegistry(client client.Client, logger logr.Logger, config Operat return nil } -func (r registry) AllOperators() map[Kind]Operator { +func (r registry) AllOperators() map[Kind]resource.Operator { return r.operators } diff --git a/internal/registry/resource/resource.go b/internal/registry/resource/resource.go new file mode 100644 index 000000000..926b47f73 --- /dev/null +++ b/internal/registry/resource/resource.go @@ -0,0 +1,40 @@ +package resource + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/go-logr/logr" +) + +const ( + ConfigMapCheckSumKey = "checksum/etcd-configmap" +) + +type OperatorContext struct { + context.Context + RunID string + Logger logr.Logger + Data map[string]string +} + +func NewOperatorContext(ctx context.Context, logger logr.Logger, runID string) OperatorContext { + return OperatorContext{ + Context: ctx, + RunID: runID, + Logger: logger, + Data: make(map[string]string), + } +} + +// Operator manages one or more resources of a specific Kind which are provisioned for an etcd cluster. +type Operator interface { + // GetExistingResourceNames gets all resources that currently exist that this Operator manages. + GetExistingResourceNames(ctx OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) + // TriggerDelete triggers the deletion of all resources that this Operator manages. + TriggerDelete(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error + // Sync synchronizes all resources that this Operator manages. If a resource does not exist then it will + // create it. If there are changes in the owning Etcd resource that transpires changes to one or more resources + // managed by this Operator then those resource(s) will be either be updated or a deletion is triggered. + Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error +} diff --git a/internal/resource/clientservice/clientservice.go b/internal/resource/clientservice/clientservice.go index 459d63c0d..485e9322a 100644 --- a/internal/resource/clientservice/clientservice.go +++ b/internal/resource/clientservice/clientservice.go @@ -3,7 +3,7 @@ package clientservice import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/resource/configmap/configmap.go b/internal/resource/configmap/configmap.go index 5504a3b3b..3ac4f4abd 100644 --- a/internal/resource/configmap/configmap.go +++ b/internal/resource/configmap/configmap.go @@ -4,7 +4,7 @@ import ( "encoding/json" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" "github.com/go-logr/logr" diff --git a/internal/resource/memberlease/memberlease.go b/internal/resource/memberlease/memberlease.go index a53dcfb42..e3d79a73c 100644 --- a/internal/resource/memberlease/memberlease.go +++ b/internal/resource/memberlease/memberlease.go @@ -5,8 +5,9 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/flow" diff --git a/internal/resource/peerservice/peerservice.go b/internal/resource/peerservice/peerservice.go index e325d8f11..be6bbffe4 100644 --- a/internal/resource/peerservice/peerservice.go +++ b/internal/resource/peerservice/peerservice.go @@ -2,7 +2,7 @@ package peerservice import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/resource/poddistruptionbudget/poddisruptionbudget.go b/internal/resource/poddistruptionbudget/poddisruptionbudget.go index 11485abdb..410fb6aea 100644 --- a/internal/resource/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/resource/poddistruptionbudget/poddisruptionbudget.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" policyv1 "k8s.io/api/policy/v1" diff --git a/internal/resource/role/role.go b/internal/resource/role/role.go index 7cfe65cc8..1fde38a80 100644 --- a/internal/resource/role/role.go +++ b/internal/resource/role/role.go @@ -2,7 +2,7 @@ package role import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" rbacv1 "k8s.io/api/rbac/v1" diff --git a/internal/resource/rolebinding/rolebinding.go b/internal/resource/rolebinding/rolebinding.go index 4097b96ab..039eab6b3 100644 --- a/internal/resource/rolebinding/rolebinding.go +++ b/internal/resource/rolebinding/rolebinding.go @@ -2,7 +2,7 @@ package rolebinding import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" rbacv1 "k8s.io/api/rbac/v1" diff --git a/internal/resource/serviceaccount/serviceaccount.go b/internal/resource/serviceaccount/serviceaccount.go index 6c2d6cc70..93ad1a111 100644 --- a/internal/resource/serviceaccount/serviceaccount.go +++ b/internal/resource/serviceaccount/serviceaccount.go @@ -2,7 +2,7 @@ package serviceaccount import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" diff --git a/internal/resource/snapshotlease/snapshotlease.go b/internal/resource/snapshotlease/snapshotlease.go index d3a4d7d39..3c4599b55 100644 --- a/internal/resource/snapshotlease/snapshotlease.go +++ b/internal/resource/snapshotlease/snapshotlease.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go index aa53ac118..7c8dd307f 100644 --- a/internal/utils/concurrent.go +++ b/internal/utils/concurrent.go @@ -5,7 +5,7 @@ import ( "runtime/debug" "sync" - "github.com/gardener/etcd-druid/internal/resource" + "github.com/gardener/etcd-druid/internal/registry/resource" ) // OperatorTask is a holder for a named function. From a41bcd4cbd5c892c3dda1e2e4caef09bc1c2c310 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 20 Nov 2023 14:10:50 +0530 Subject: [PATCH 007/235] Fixed lint error --- api/v1alpha1/types_etcd.go | 1 + .../crds/templates/10-crd-druid.gardener.cloud_etcds.yaml | 2 ++ config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index da3bd3837..4bb79bc1f 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -468,6 +468,7 @@ type LastOperation struct { // ErrorCode is a string alias representing an error code that identifies an error. type ErrorCode string +// LastError stores details of the most recent error encountered for a resource. type LastError struct { // Code is an error code that uniquely identifies an error. Code ErrorCode `json:"code"` diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index cb75fb2a9..cdbce78a8 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -1841,6 +1841,8 @@ spec: description: LastErrors captures errors that occurred during the last operation. items: + description: LastError stores details of the most recent error encountered + for a resource. properties: code: description: Code is an error code that uniquely identifies diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index cb75fb2a9..cdbce78a8 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -1841,6 +1841,8 @@ spec: description: LastErrors captures errors that occurred during the last operation. items: + description: LastError stores details of the most recent error encountered + for a resource. properties: code: description: Code is an error code that uniquely identifies From c7509e60598d9d0924cc7687f236c39b956a46c3 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 20 Nov 2023 16:45:01 +0530 Subject: [PATCH 008/235] renamed resource package to operator --- internal/controller/compaction/config.go | 36 +- internal/controller/compaction/reconciler.go | 570 +++++------------- internal/controller/etcd/reconcile_delete.go | 2 +- internal/controller/etcd/reconciler.go | 10 +- internal/controller/utils/etcdstatus.go | 2 +- internal/controller/utils/reconciler.go | 21 + .../clientservice/clientservice.go | 0 .../configmap/configmap.go | 0 .../configmap/etcdconfig.go | 0 .../memberlease/memberlease.go | 0 .../peerservice/peerservice.go | 0 .../poddisruptionbudget.go | 2 +- internal/{registry => operator}/registry.go | 29 +- .../resource/types.go} | 0 internal/{resource => operator}/role/role.go | 0 .../rolebinding/rolebinding.go | 0 .../serviceaccount/serviceaccount.go | 0 .../snapshotlease/snapshotlease.go | 0 internal/utils/concurrent.go | 2 +- internal/utils/miscellaneous.go | 16 +- 20 files changed, 221 insertions(+), 469 deletions(-) rename internal/{resource => operator}/clientservice/clientservice.go (100%) rename internal/{resource => operator}/configmap/configmap.go (100%) rename internal/{resource => operator}/configmap/etcdconfig.go (100%) rename internal/{resource => operator}/memberlease/memberlease.go (100%) rename internal/{resource => operator}/peerservice/peerservice.go (100%) rename internal/{resource => operator}/poddistruptionbudget/poddisruptionbudget.go (97%) rename internal/{registry => operator}/registry.go (70%) rename internal/{registry/resource/resource.go => operator/resource/types.go} (100%) rename internal/{resource => operator}/role/role.go (100%) rename internal/{resource => operator}/rolebinding/rolebinding.go (100%) rename internal/{resource => operator}/serviceaccount/serviceaccount.go (100%) rename internal/{resource => operator}/snapshotlease/snapshotlease.go (100%) diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 0c7d3e634..fe0fd70b9 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -29,18 +29,24 @@ var featureList = []featuregate.Feature{ features.UseEtcdWrapper, } +// Flag name constants const ( - enableBackupCompactionFlagName = "enable-backup-compaction" - workersFlagName = "compaction-workers" - eventsThresholdFlagName = "etcd-events-threshold" - activeDeadlineDurationFlagName = "active-deadline-duration" - metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" + enableBackupCompactionFlagName = "enable-backup-compaction" + workersFlagName = "compaction-workers" + eventsThresholdFlagName = "etcd-events-threshold" + activeDeadlineDurationFlagName = "active-deadline-duration" + cleanupDeadlineDurationAfterCompletionFlagName = "cleanup-deadline-duration-after-completion" + metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" +) - defaultEnableBackupCompaction = false - defaultCompactionWorkers = 3 - defaultEventsThreshold = 1000000 - defaultActiveDeadlineDuration = 3 * time.Hour - defaultMetricsScrapeWaitDuration = 0 +// Default value constants +const ( + defaultEnableBackupCompaction = false + defaultCompactionWorkers = 3 + defaultEventsThreshold = 1000000 + defaultActiveDeadlineDuration = 3 * time.Hour + defaultCleanupDeadlineDurationAfterCompletion = 3 * time.Hour + defaultMetricsScrapeWaitDuration = 0 ) // Config contains configuration for the Compaction Controller. @@ -53,6 +59,10 @@ type Config struct { EventsThreshold int64 // ActiveDeadlineDuration is the duration after which a running compaction job will be killed. ActiveDeadlineDuration time.Duration + // CleanupDeadlineAfterCompletion is the duration after which an explicit cleanup of resources used by compaction will be done. + // This will ensure that resources used by compaction are eventually cleaned up. This additionally ensures that an operator + // can inspect the resources before they are cleaned-up. + CleanupDeadlineDurationAfterCompletion time.Duration // MetricsScrapeWaitDuration is the duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped MetricsScrapeWaitDuration time.Duration // FeatureGates contains the feature gates to be used by Compaction Controller. @@ -68,9 +78,11 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.Int64Var(&cfg.EventsThreshold, eventsThresholdFlagName, defaultEventsThreshold, "Total number of etcd events that can be allowed before a backup compaction job is triggered.") fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, - "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\").") + "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\" or similar).") + fs.DurationVar(&cfg.CleanupDeadlineDurationAfterCompletion, cleanupDeadlineDurationAfterCompletionFlagName, defaultCleanupDeadlineDurationAfterCompletion, + "Duration after which resources consumed by a completed compaction is cleaned up (Ex: \\\"300ms\\\" or \\\"2h45m\\\" or similar).\\\").") fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagname, defaultMetricsScrapeWaitDuration, - "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\").") + "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\" or similar).") } // Validate validates the config. diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index cab28f1d6..5ccf26755 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -16,26 +16,19 @@ package compaction import ( "context" - "fmt" - "strconv" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - druidmetrics "github.com/gardener/etcd-druid/internal/metrics" "github.com/gardener/etcd-druid/internal/utils" - "github.com/gardener/etcd-druid/pkg/features" + druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" "github.com/gardener/gardener/pkg/utils/imagevector" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -43,13 +36,13 @@ import ( ) const ( - // DefaultETCDQuota is the default etcd quota. - DefaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi + // defaultETCDQuota is the default etcd quota. + defaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi ) // Reconciler reconciles compaction jobs for Etcd resources. type Reconciler struct { - client.Client + client client.Client config *Config imageVector imagevector.ImageVector logger logr.Logger @@ -65,13 +58,12 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { } // NewReconcilerWithImageVector creates a new reconciler for Compaction with an ImageVector. -// This constructor will mostly be used by tests. func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { return &Reconciler{ - Client: mgr.GetClient(), + client: mgr.GetClient(), config: config, imageVector: imageVector, - logger: log.Log.WithName("compaction-lease-controller"), + logger: log.Log.WithName(controllerName), } } @@ -82,477 +74,205 @@ func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVect // Reconcile reconciles the compaction job. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - r.logger.Info("Compaction job reconciliation started") etcd := &druidv1alpha1.Etcd{} - if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { - if errors.IsNotFound(err) { - // Object not found, return. Created objects are automatically garbage collected. - // For additional cleanup logic use finalizers. - return ctrl.Result{}, nil - } - // Error reading the object - requeue the request. - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err + if result := ctrlutils.GetLatestEtcd(ctx, r.client, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() } - - if !etcd.DeletionTimestamp.IsZero() || etcd.Spec.Backup.Store == nil { - // Delete compaction job if exists - return r.delete(ctx, r.logger, etcd) + rLog := r.logger.WithValues("etcd", etcd.GetNamespaceName()) + jobKey := getJobKey(etcd) + if etcd.IsMarkedForDeletion() || !etcd.IsBackupEnabled() { + return r.triggerJobDeletion(ctx, rLog, jobKey) } - logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) - - return r.reconcileJob(ctx, logger, etcd) -} - -func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { - // Update metrics for currently running compaction job, if any - job := &batchv1.Job{} - if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { - if errors.IsNotFound(err) { - logger.Info("Currently, no compaction job is running in the namespace ", etcd.Namespace) - } else { - // Error reading the object - requeue the request. - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", etcd.GetCompactionJobName(), etcd.Namespace, err) - } + if result := r.reconcileExistingJob(ctx, rLog, jobKey); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() } - if job != nil && job.Name != "" { - if !job.DeletionTimestamp.IsZero() { - logger.Info("Job is already in deletion. A new job will be created only if the previous one has been deleted.", "namespace: ", job.Namespace, "name: ", job.Name) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, nil - } + return r.reconcileJob(ctx, rLog) + /* + Get latest etcd + if it is not found then do not requeue + if there is an error then requeue with error - // Check if there is one active job or not - if job.Status.Active > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) - // Don't need to requeue if the job is currently running - return ctrl.Result{}, nil + if it's marked for deletion { + cleanup any Jobs still out there } + Get Existing Job + if Job not found then continue + If error in getting Job requeue with error + If Job exists and is in deletion, requeue + If Job is active return with no-requeue + If Job succeeded record metrics and delete job and continue reconcile + If Job failed record metrics, delete job and continue reconcile + Get delta and full leases + If error requeue with error + compute difference and if difference > eventsThreshold create Job and record metrics + */ - // Delete job if the job succeeded - if job.Status.Succeeded > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) - if job.Status.CompletionTime != nil { - metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) - } - if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { - logger.Error(err, "Couldn't delete the successful job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while deleting successful compaction job: %v", err) - } - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() - } - - // Delete job and requeue if the job failed - if job.Status.Failed > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) - if job.Status.StartTime != nil { - metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) - } - err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) - if err != nil { - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while deleting failed compaction job: %v", err) - } - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, nil - } - } - - // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job - fullLease := &coordinationv1.Lease{} - - if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetFullSnapshotLeaseName()), fullLease); err != nil { - logger.Error(err, "Couldn't fetch full snap lease", "namespace", etcd.Namespace, "name", etcd.GetFullSnapshotLeaseName()) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } - - deltaLease := &coordinationv1.Lease{} - if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetDeltaSnapshotLeaseName()), deltaLease); err != nil { - logger.Error(err, "Couldn't fetch delta snap lease", "namespace", etcd.Namespace, "name", etcd.GetDeltaSnapshotLeaseName()) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } +} - // Revisions have not been set yet by etcd-back-restore container. - // Skip further processing as we cannot calculate a revision delta. - if fullLease.Spec.HolderIdentity == nil || deltaLease.Spec.HolderIdentity == nil { - return ctrl.Result{}, nil +func getJobKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{ + Namespace: etcd.Namespace, + Name: etcd.GetCompactionJobName(), } +} - full, err := strconv.ParseInt(*fullLease.Spec.HolderIdentity, 10, 64) - if err != nil { - logger.Error(err, "Can't convert holder identity of full snap lease to integer", - "namespace", fullLease.Namespace, "leaseName", fullLease.Name, "holderIdentity", fullLease.Spec.HolderIdentity) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err +func (r *Reconciler) reconcileExistingJob(ctx context.Context, logger logr.Logger, jobKey client.ObjectKey) ctrlutils.ReconcileStepResult { + rjLog := logger.WithValues("operation", "reconcile-compaction-job", "job", jobKey) + job := &batchv1.Job{} + if err := r.client.Get(ctx, jobKey, job); !errors.IsNotFound(err) { + return ctrlutils.ReconcileWithError(err) } - delta, err := strconv.ParseInt(*deltaLease.Spec.HolderIdentity, 10, 64) - if err != nil { - logger.Error(err, "Can't convert holder identity of delta snap lease to integer", - "namespace", deltaLease.Namespace, "leaseName", deltaLease.Name, "holderIdentity", deltaLease.Spec.HolderIdentity) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err + type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult + handlers := []existingJobHandler{ + r.handleJobDeletionInProgress, + r.handleActiveJob, + r.handleSuccessfulJobCompletion, + r.handleFailedJobCompletion, } - - diff := delta - full - metricNumDeltaEvents.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(float64(diff)) - - // Reconcile job only when number of accumulated revisions over the last full snapshot is more than the configured threshold value via 'events-threshold' flag - if diff >= r.config.EventsThreshold { - logger.Info("Creating etcd compaction job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) - job, err = r.createCompactionJob(ctx, logger, etcd) - if err != nil { - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error during compaction job creation: %v", err) + for _, handler := range handlers { + if result := handler(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { + return result } - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) - } - - if job.Name != "" { - logger.Info("Current compaction job status", - "namespace", job.Namespace, "name", job.Name, "succeeded", job.Status.Succeeded) } + return ctrlutils.ContinueReconcile() +} - return ctrl.Result{Requeue: false}, nil +func (r *Reconciler) getMatchingJobs(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { + r.client.List(ctx) } -func (r *Reconciler) delete(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { +func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { + jobObjectKey := client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetCompactionJobName()} + // Get any existing job that might exist at this time job := &batchv1.Job{} - err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job) - if err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction job: %v", err) - } - return ctrl.Result{Requeue: false}, nil + if err := r.client.Get(ctx, jobObjectKey, job); !errors.IsNotFound(err) { + return ctrlutils.ReconcileWithError(err).ReconcileResult() } - if job.DeletionTimestamp == nil { - logger.Info("Deleting job", "namespace", job.Namespace, "name", job.Name) - if err := client.IgnoreNotFound(r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while deleting compaction job: %v", err) + // If there is an existing Job then check the Job status and take appropriate action. + if !utils.IsEmptyString(job.Name) { + if result := r.handleExistingJob(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() } } - logger.Info("No compaction job is running") - return ctrl.Result{ - Requeue: false, - }, nil -} - -func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*batchv1.Job, error) { - activeDeadlineSeconds := r.config.ActiveDeadlineDuration.Seconds() - - _, etcdBackupImage, _, err := utils.GetEtcdImages(etcd, r.imageVector, r.config.FeatureGates[features.UseEtcdWrapper]) - if err != nil { - return nil, fmt.Errorf("couldn't fetch etcd backup image: %v", err) - } - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetCompactionJobName(), - Namespace: etcd.Namespace, - Labels: getLabels(etcd), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: druidv1alpha1.GroupVersion.String(), - BlockOwnerDeletion: pointer.Bool(true), - Controller: pointer.Bool(true), - Kind: "Etcd", - Name: etcd.Name, - UID: etcd.UID, - }, - }, - }, - - Spec: batchv1.JobSpec{ - ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), - Completions: pointer.Int32(1), - BackoffLimit: pointer.Int32(0), - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: etcd.Spec.Annotations, - Labels: getLabels(etcd), - }, - Spec: v1.PodSpec{ - ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), - ServiceAccountName: etcd.GetServiceAccountName(), - RestartPolicy: v1.RestartPolicyNever, - Containers: []v1.Container{{ - Name: "compact-backup", - Image: *etcdBackupImage, - ImagePullPolicy: v1.PullIfNotPresent, - Args: getCompactionJobArgs(etcd, r.config.MetricsScrapeWaitDuration.String()), - }}, - }, - }, - }, - } + latestDeltaRevision, err := getLatestDeltaRevision(ctx, etcd.GetDeltaSnapshotLeaseName()) - if vms, err := getCompactionJobVolumeMounts(etcd); err != nil { - return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", - etcd.Namespace, - etcd.Name, - err) - } else { - job.Spec.Template.Spec.Containers[0].VolumeMounts = vms - } +} - if env, err := getCompactionJobEnvVar(etcd); err != nil { - return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", - etcd.Namespace, - etcd.Name, - err) - } else { - job.Spec.Template.Spec.Containers[0].Env = env - } +func (r *Reconciler) canScheduleJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { + r.getLatestFullSnapshotRevision(ctx, logger, etcd) +} - if vm, err := getCompactionJobVolumes(ctx, r.Client, r.logger, etcd); err != nil { - return nil, fmt.Errorf("error creating compaction job in %v for %v : %v", - etcd.Namespace, - etcd.Name, - err) - } else { - job.Spec.Template.Spec.Volumes = vm +func (r *Reconciler) getLatestFullSnapshotRevision(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (int64, error) { + fullLease := &coordinationv1.Lease{} + if err := r.client.Get(ctx, client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetFullSnapshotLeaseName()}, fullLease); err != nil { + logger.Error(err, "could not fetch full snapshot lease", "lease-name", etcd.GetFullSnapshotLeaseName()) + return -1, err } + return parseRevision(fullLease) +} - if etcd.Spec.Backup.CompactionResources != nil { - job.Spec.Template.Spec.Containers[0].Resources = *etcd.Spec.Backup.CompactionResources - } +func parseRevision(lease *coordinationv1.Lease) (int64, error) { + if lease.Spec.HolderIdentity == nil { - logger.Info("Creating job", "namespace", job.Namespace, "name", job.Name) - err = r.Create(ctx, job) - if err != nil { - return nil, err } - - //TODO (abdasgupta): Evaluate necessity of claiming object here after creation - return job, nil } -func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { - return map[string]string{ - "name": "etcd-backup-compaction", - "instance": etcd.Name, - "gardener.cloud/role": "controlplane", - "networking.gardener.cloud/to-dns": "allowed", - "networking.gardener.cloud/to-private-networks": "allowed", - "networking.gardener.cloud/to-public-networks": "allowed", +func (r *Reconciler) handleExistingJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { + type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult + handlers := []existingJobHandler{ + r.handleJobDeletionInProgress, + r.handleActiveJob, + r.handleSuccessfulJobCompletion, + r.handleFailedJobCompletion, + } + for _, handler := range handlers { + if result := handler(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } } + return ctrlutils.ContinueReconcile() } -func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd) ([]v1.VolumeMount, error) { - vms := []v1.VolumeMount{ - { - Name: "etcd-workspace-dir", - MountPath: "/var/etcd/data", - }, - } - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err != nil { - return vms, fmt.Errorf("storage provider is not recognized while fetching volume mounts") - } - switch provider { - case utils.Local: - vms = append(vms, v1.VolumeMount{ - Name: "host-storage", - MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), - }) - case utils.GCS: - vms = append(vms, v1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/.gcp/", - }) - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - vms = append(vms, v1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/etcd-backup/", - }) +func (r *Reconciler) handleJobDeletionInProgress(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { + if !job.DeletionTimestamp.IsZero() { + logger.Info("Deletion has been triggered for the job. A new job can only be created once the previously scheduled job has been deleted") + return ctrlutils.ReconcileAfter(10*time.Second, "deletion in progress, requeuing job") } + return ctrlutils.ContinueReconcile() +} - return vms, nil +func (r *Reconciler) handleActiveJob(_ context.Context, _ logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(1) + return ctrlutils.DoNotRequeue() } -func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr.Logger, etcd *druidv1alpha1.Etcd) ([]v1.Volume, error) { - vs := []v1.Volume{ - { - Name: "etcd-workspace-dir", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, +func (r *Reconciler) handleFailedJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { + if job.Status.Failed > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) + if job.Status.StartTime != nil { + metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: job.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) + } } + return ctrlutils.ContinueReconcile() +} - storeValues := etcd.Spec.Backup.Store - provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) - if err != nil { - return vs, fmt.Errorf("could not recognize storage provider while fetching volumes") - } - switch provider { - case "Local": - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, etcd.Namespace) - if err != nil { - return vs, fmt.Errorf("could not determine host mount path for local provider") +func (r *Reconciler) handleSuccessfulJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { + if job.Status.Succeeded > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) + if job.Status.CompletionTime != nil { + metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) } - - hpt := v1.HostPathDirectory - vs = append(vs, v1.Volume{ - Name: "host-storage", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), - Type: &hpt, - }, - }, - }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: - if storeValues.SecretRef == nil { - return vs, fmt.Errorf("could not configure secretRef for backup store %v", provider) + if result := r.deleteJob(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { + return result } - - vs = append(vs, v1.Volume{ - Name: "etcd-backup", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, - }, - }, - }) + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Inc() } - - return vs, nil + return ctrlutils.ContinueReconcile() } -func getCompactionJobEnvVar(etcd *druidv1alpha1.Etcd) ([]v1.EnvVar, error) { - var env []v1.EnvVar - - storeValues := etcd.Spec.Backup.Store - - env = append(env, getEnvVarFromValues("STORAGE_CONTAINER", *storeValues.Container)) - env = append(env, getEnvVarFromFields("POD_NAMESPACE", "metadata.namespace")) - - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err != nil { - return env, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") - } - - switch provider { - case utils.S3: - env = append(env, getEnvVarFromValues("AWS_APPLICATION_CREDENTIALS", "/var/etcd-backup")) - case utils.ABS: - env = append(env, getEnvVarFromValues("AZURE_APPLICATION_CREDENTIALS", "/var/etcd-backup")) - case utils.GCS: - env = append(env, getEnvVarFromValues("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) - case utils.Swift: - env = append(env, getEnvVarFromValues("OPENSTACK_APPLICATION_CREDENTIALS", "/var/etcd-backup")) - case utils.OSS: - env = append(env, getEnvVarFromValues("ALICLOUD_APPLICATION_CREDENTIALS", "/var/etcd-backup")) - case utils.ECS: - if storeValues.SecretRef == nil { - return env, fmt.Errorf("no secretRef could be configured for backup store of ECS") +func (r *Reconciler) triggerJobDeletion(ctx context.Context, logger logr.Logger, jobObjectKey client.ObjectKey) (ctrl.Result, error) { + dLog := logger.WithValues("operation", "delete-compaction-job", "job", jobObjectKey) + job := &batchv1.Job{} + if err := r.client.Get(ctx, jobObjectKey, job); err != nil { + if errors.IsNotFound(err) { + dLog.Info("No compaction job exists, nothing to clean up") + return ctrlutils.DoNotRequeue().ReconcileResult() } - - env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) - env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) - env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) - case utils.OCS: - env = append(env, getEnvVarFromValues("OPENSHIFT_APPLICATION_CREDENTIALS", "/var/etcd-backup")) + return ctrlutils.ReconcileWithError(err).ReconcileResult() } - return env, nil -} - -func getEnvVarFromValues(name, value string) v1.EnvVar { - return v1.EnvVar{ - Name: name, - Value: value, + if job.DeletionTimestamp != nil { + dLog.Info("Deletion has already been triggered for the compaction job. Skipping further action.") + return ctrlutils.DoNotRequeue().ReconcileResult() } -} -func getEnvVarFromFields(name, fieldPath string) v1.EnvVar { - return v1.EnvVar{ - Name: name, - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: fieldPath, - }, - }, + dLog.Info("Triggering delete of compaction job") + if result := r.deleteJob(ctx, dLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() } + return ctrlutils.DoNotRequeue().ReconcileResult() } -func getEnvVarFromSecrets(name, secretName, secretKey string) v1.EnvVar { - return v1.EnvVar{ - Name: name, - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: secretName, - }, - Key: secretKey, - }, - }, +func (r *Reconciler) deleteJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { + if err := client.IgnoreNotFound(r.client.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { + logger.Error(err, "error when deleting compaction job") + return ctrlutils.ReconcileWithError(err) } + return ctrlutils.ContinueReconcile() } -func getCompactionJobArgs(etcd *druidv1alpha1.Etcd, metricsScrapeWaitDuration string) []string { - command := []string{"compact"} - command = append(command, "--data-dir=/var/etcd/data/compaction.etcd") - command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp") - command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") - command = append(command, "--metrics-scrape-wait-duration="+metricsScrapeWaitDuration) - command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) - command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) - - var quota int64 = DefaultETCDQuota - if etcd.Spec.Etcd.Quota != nil { - quota = etcd.Spec.Etcd.Quota.Value() - } - command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) - - if etcd.Spec.Etcd.EtcdDefragTimeout != nil { - command = append(command, "--etcd-defrag-timeout="+etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String()) - } - - backupValues := etcd.Spec.Backup - if backupValues.EtcdSnapshotTimeout != nil { - command = append(command, "--etcd-snapshot-timeout="+backupValues.EtcdSnapshotTimeout.Duration.String()) - } - storeValues := etcd.Spec.Backup.Store - if storeValues != nil { - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err == nil { - command = append(command, "--storage-provider="+provider) - } - - if storeValues.Prefix != "" { - command = append(command, "--store-prefix="+storeValues.Prefix) - } - - if storeValues.Container != nil { - command = append(command, "--store-container="+*(storeValues.Container)) +func (r *Reconciler) getLatestJob(ctx context.Context, objectKey client.ObjectKey) (*batchv1.Job, error) { + job := &batchv1.Job{} + if err := r.client.Get(ctx, objectKey, job); err != nil { + if errors.IsNotFound(err) { + return nil, nil } + return nil, err } - - return command + return job, nil } diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index fd2e0b136..a7da6c643 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -7,7 +7,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index ff67cff61..7229a84e5 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -5,8 +5,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/registry" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" "github.com/google/uuid" @@ -24,7 +24,7 @@ type Reconciler struct { config *Config recorder record.EventRecorder imageVector imagevector.ImageVector - operatorRegistry registry.OperatorRegistry + operatorRegistry operator.Registry lastOpErrRecorder ctrlutils.LastOperationErrorRecorder logger logr.Logger } @@ -36,9 +36,9 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { if err != nil { return nil, err } - operatorReg := registry.NewOperatorRegistry(mgr.GetClient(), + operatorReg := operator.NewRegistry(mgr.GetClient(), logger, - registry.OperatorConfig{ + operator.Config{ DisableEtcdServiceAccountAutomount: config.DisableEtcdServiceAccountAutomount, }, ) diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index 437cba2b9..978b7c067 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 1189c02ce..7c688936e 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -15,13 +15,16 @@ package utils import ( + "context" "errors" "fmt" "path/filepath" "time" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/gardener/pkg/utils/imagevector" + apierrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -60,6 +63,16 @@ func ContainsFinalizer(o client.Object, finalizer string) bool { return false } +func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ReconcileStepResult { + if err := client.Get(ctx, objectKey, etcd); err != nil { + if apierrors.IsNotFound(err) { + return DoNotRequeue() + } + return ReconcileWithError(err) + } + return ContinueReconcile() +} + type ReconcileStepResult struct { result ctrl.Result errs []error @@ -111,6 +124,14 @@ func ReconcileAfter(period time.Duration, description string) ReconcileStepResul } } +func ReconcileWithErrorAfter(period time.Duration, errs ...error) ReconcileStepResult { + return ReconcileStepResult{ + result: ctrl.Result{RequeueAfter: period}, + errs: errs, + continueReconcile: false, + } +} + func ShortCircuitReconcileFlow(result ReconcileStepResult) bool { return !result.continueReconcile } diff --git a/internal/resource/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go similarity index 100% rename from internal/resource/clientservice/clientservice.go rename to internal/operator/clientservice/clientservice.go diff --git a/internal/resource/configmap/configmap.go b/internal/operator/configmap/configmap.go similarity index 100% rename from internal/resource/configmap/configmap.go rename to internal/operator/configmap/configmap.go diff --git a/internal/resource/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go similarity index 100% rename from internal/resource/configmap/etcdconfig.go rename to internal/operator/configmap/etcdconfig.go diff --git a/internal/resource/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go similarity index 100% rename from internal/resource/memberlease/memberlease.go rename to internal/operator/memberlease/memberlease.go diff --git a/internal/resource/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go similarity index 100% rename from internal/resource/peerservice/peerservice.go rename to internal/operator/peerservice/peerservice.go diff --git a/internal/resource/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go similarity index 97% rename from internal/resource/poddistruptionbudget/poddisruptionbudget.go rename to internal/operator/poddistruptionbudget/poddisruptionbudget.go index 410fb6aea..591827395 100644 --- a/internal/resource/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" policyv1 "k8s.io/api/policy/v1" diff --git a/internal/registry/registry.go b/internal/operator/registry.go similarity index 70% rename from internal/registry/registry.go rename to internal/operator/registry.go index b1d3d3657..3d35b9cae 100644 --- a/internal/registry/registry.go +++ b/internal/operator/registry.go @@ -1,26 +1,25 @@ -package registry +package operator import ( - "github.com/gardener/etcd-druid/internal/registry/resource" - "github.com/gardener/etcd-druid/internal/resource/clientservice" - "github.com/gardener/etcd-druid/internal/resource/configmap" - "github.com/gardener/etcd-druid/internal/resource/memberlease" - "github.com/gardener/etcd-druid/internal/resource/peerservice" - "github.com/gardener/etcd-druid/internal/resource/poddistruptionbudget" - "github.com/gardener/etcd-druid/internal/resource/role" - "github.com/gardener/etcd-druid/internal/resource/rolebinding" - "github.com/gardener/etcd-druid/internal/resource/serviceaccount" - "github.com/gardener/etcd-druid/internal/resource/snapshotlease" - + "github.com/gardener/etcd-druid/internal/operator/clientservice" + "github.com/gardener/etcd-druid/internal/operator/configmap" + "github.com/gardener/etcd-druid/internal/operator/memberlease" + "github.com/gardener/etcd-druid/internal/operator/peerservice" + "github.com/gardener/etcd-druid/internal/operator/poddistruptionbudget" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/role" + "github.com/gardener/etcd-druid/internal/operator/rolebinding" + "github.com/gardener/etcd-druid/internal/operator/serviceaccount" + "github.com/gardener/etcd-druid/internal/operator/snapshotlease" "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" ) -type OperatorConfig struct { +type Config struct { DisableEtcdServiceAccountAutomount bool } -type OperatorRegistry interface { +type Registry interface { AllOperators() map[Kind]resource.Operator StatefulSetOperator() resource.Operator ServiceAccountOperator() resource.Operator @@ -53,7 +52,7 @@ type registry struct { operators map[Kind]resource.Operator } -func NewOperatorRegistry(client client.Client, logger logr.Logger, config OperatorConfig) OperatorRegistry { +func NewRegistry(client client.Client, logger logr.Logger, config Config) Registry { operators := make(map[Kind]resource.Operator) operators[ConfigMapKind] = configmap.New(client, logger) operators[ServiceAccountKind] = serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount) diff --git a/internal/registry/resource/resource.go b/internal/operator/resource/types.go similarity index 100% rename from internal/registry/resource/resource.go rename to internal/operator/resource/types.go diff --git a/internal/resource/role/role.go b/internal/operator/role/role.go similarity index 100% rename from internal/resource/role/role.go rename to internal/operator/role/role.go diff --git a/internal/resource/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go similarity index 100% rename from internal/resource/rolebinding/rolebinding.go rename to internal/operator/rolebinding/rolebinding.go diff --git a/internal/resource/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go similarity index 100% rename from internal/resource/serviceaccount/serviceaccount.go rename to internal/operator/serviceaccount/serviceaccount.go diff --git a/internal/resource/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go similarity index 100% rename from internal/resource/snapshotlease/snapshotlease.go rename to internal/operator/snapshotlease/snapshotlease.go diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go index 7c8dd307f..69cda6c5a 100644 --- a/internal/utils/concurrent.go +++ b/internal/utils/concurrent.go @@ -5,7 +5,7 @@ import ( "runtime/debug" "sync" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" ) // OperatorTask is a holder for a named function. diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index 3ab005a15..5077ea04a 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -17,6 +17,7 @@ package utils import ( "fmt" "maps" + "strings" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -85,14 +86,6 @@ func Key(namespaceOrName string, nameOpt ...string) client.ObjectKey { return client.ObjectKey{Namespace: namespace, Name: name} } -// Max returns the larger of x or y. -func Max(x, y int) int { - if y > x { - return y - } - return x -} - // TypeDeref dereferences a pointer to a type if it is not nil, else it returns the default value. func TypeDeref[T any](val *T, defaultVal T) T { if val != nil { @@ -100,3 +93,10 @@ func TypeDeref[T any](val *T, defaultVal T) T { } return defaultVal } + +func IsEmptyString(s string) bool { + if len(strings.TrimSpace(s)) == 0 { + return true + } + return false +} From f977da0a9f857bdcd8f4205a6a8d3b5e2e86f3bb Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Thu, 23 Nov 2023 10:28:24 +0530 Subject: [PATCH 009/235] fixed import error --- internal/operator/clientservice/clientservice.go | 2 +- internal/operator/configmap/configmap.go | 2 +- internal/operator/memberlease/memberlease.go | 2 +- internal/operator/peerservice/peerservice.go | 2 +- internal/operator/role/role.go | 2 +- internal/operator/rolebinding/rolebinding.go | 2 +- internal/operator/serviceaccount/serviceaccount.go | 2 +- internal/operator/snapshotlease/snapshotlease.go | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 485e9322a..4af37aac5 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -3,7 +3,7 @@ package clientservice import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 3ac4f4abd..e5fd6942b 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -4,7 +4,7 @@ import ( "encoding/json" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" "github.com/go-logr/logr" diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index e3d79a73c..3ca787582 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index be6bbffe4..e94361e6a 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -2,7 +2,7 @@ package peerservice import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 1fde38a80..1599276e4 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -2,7 +2,7 @@ package role import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" rbacv1 "k8s.io/api/rbac/v1" diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index 039eab6b3..27ad7de4d 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -2,7 +2,7 @@ package rolebinding import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" rbacv1 "k8s.io/api/rbac/v1" diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 93ad1a111..c436bcdaa 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -2,7 +2,7 @@ package serviceaccount import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 3c4599b55..6d5d414ed 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/registry/resource" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" From ccf9ed17bf413827a6d6b35cb1bd8afa4f9a102f Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 23 Nov 2023 16:59:44 +0530 Subject: [PATCH 010/235] Add `DataVolumesReady` condition, update imports for internal/health packages --- api/v1alpha1/types_etcd.go | 3 +- .../health/condition/check_all_members.go | 4 +- .../condition/check_all_members_test.go | 10 +-- .../condition/check_backup_ready_test.go | 2 +- .../condition/check_data_volumes_ready.go | 70 +++++++++++++++++++ internal/health/condition/check_ready_test.go | 2 +- internal/health/etcdmember/builder_test.go | 8 +-- internal/health/etcdmember/check_ready.go | 9 ++- .../health/etcdmember/check_ready_test.go | 15 ++-- internal/health/status/check.go | 9 +-- internal/health/status/check_test.go | 20 +++--- internal/utils/statefulset.go | 35 ++++++++++ 12 files changed, 147 insertions(+), 40 deletions(-) create mode 100644 internal/health/condition/check_data_volumes_ready.go diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 4bb79bc1f..c84db86a2 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -14,7 +14,6 @@ import ( "k8s.io/utils/pointer" ) -// TODO Remove unused constants const ( // GarbageCollectionPolicyExponential defines the exponential policy for garbage collecting old backups GarbageCollectionPolicyExponential = "Exponential" @@ -325,6 +324,8 @@ const ( ConditionTypeAllMembersReady ConditionType = "AllMembersReady" // ConditionTypeBackupReady is a constant for a condition type indicating that the etcd backup is ready. ConditionTypeBackupReady ConditionType = "BackupReady" + // ConditionTypeDataVolumesReady is a constant for a condition type indicating that the etcd data volumes are ready. + ConditionTypeDataVolumesReady ConditionType = "DataVolumesReady" ) // EtcdMemberConditionStatus is the status of an etcd cluster member. diff --git a/internal/health/condition/check_all_members.go b/internal/health/condition/check_all_members.go index b6f270cbb..d0863b7d5 100644 --- a/internal/health/condition/check_all_members.go +++ b/internal/health/condition/check_all_members.go @@ -63,7 +63,7 @@ func (a *allMembersReady) Check(_ context.Context, etcd druidv1alpha1.Etcd) Resu return result } -// AllMembersCheck returns a check for the "AllMembersReady" condition. -func AllMembersCheck(_ client.Client) Checker { +// AllMembersReadyCheck returns a check for the "AllMembersReady" condition. +func AllMembersReadyCheck(_ client.Client) Checker { return &allMembersReady{} } diff --git a/internal/health/condition/check_all_members_test.go b/internal/health/condition/check_all_members_test.go index d62967558..4187ac387 100644 --- a/internal/health/condition/check_all_members_test.go +++ b/internal/health/condition/check_all_members_test.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" + . "github.com/gardener/etcd-druid/internal/health/condition" ) var _ = Describe("AllMembersReadyCheck", func() { @@ -51,7 +51,7 @@ var _ = Describe("AllMembersReadyCheck", func() { }, }, } - check := AllMembersCheck(nil) + check := AllMembersReadyCheck(nil) result := check.Check(context.TODO(), etcd) @@ -72,7 +72,7 @@ var _ = Describe("AllMembersReadyCheck", func() { }, }, } - check := AllMembersCheck(nil) + check := AllMembersReadyCheck(nil) result := check.Check(context.TODO(), etcd) @@ -92,7 +92,7 @@ var _ = Describe("AllMembersReadyCheck", func() { }, }, } - check := AllMembersCheck(nil) + check := AllMembersReadyCheck(nil) result := check.Check(context.TODO(), etcd) @@ -112,7 +112,7 @@ var _ = Describe("AllMembersReadyCheck", func() { Members: []druidv1alpha1.EtcdMemberStatus{}, }, } - check := AllMembersCheck(nil) + check := AllMembersReadyCheck(nil) result := check.Check(context.TODO(), etcd) diff --git a/internal/health/condition/check_backup_ready_test.go b/internal/health/condition/check_backup_ready_test.go index 5e55c7224..064bdeb02 100644 --- a/internal/health/condition/check_backup_ready_test.go +++ b/internal/health/condition/check_backup_ready_test.go @@ -19,7 +19,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" + . "github.com/gardener/etcd-druid/internal/health/condition" mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" "github.com/golang/mock/gomock" diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go new file mode 100644 index 000000000..63f9614f4 --- /dev/null +++ b/internal/health/condition/check_data_volumes_ready.go @@ -0,0 +1,70 @@ +// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package condition + +import ( + "context" + "fmt" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type dataVolumesReady struct { + cl client.Client +} + +func (d *dataVolumesReady) Check(ctx context.Context, etcd druidv1alpha1.Etcd) Result { + res := &result{ + conType: druidv1alpha1.ConditionTypeDataVolumesReady, + status: druidv1alpha1.ConditionUnknown, + } + + sts, err := utils.GetStatefulSet(ctx, d.cl, &etcd) + if err != nil { + res.reason = "UnableToFetchStatefulSet" + res.message = fmt.Sprintf("Unable to fetch StatefulSet for etcd: %s", err.Error()) + return res + } + + pvcEvents, err := utils.FetchPVCWarningEventsForStatefulSet(ctx, d.cl, sts) + if err != nil { + res.reason = "UnableToFetchWarningEventsForDataVolumes" + res.message = fmt.Sprintf("Unable to fetch warning events for PVCs used by StatefulSet %v: %s", kutil.Key(sts.Name, sts.Namespace), err.Error()) + return res + } + + if pvcEvents != "" { + res.reason = "FoundWarningsForDataVolumes" + res.message = pvcEvents + res.status = druidv1alpha1.ConditionFalse + return res + } + + res.reason = "NoWarningsFoundForDataVolumes" + res.message = fmt.Sprintf("No warning events found for PVCs used by StatefulSet %v", kutil.Key(sts.Name, sts.Namespace)) + res.status = druidv1alpha1.ConditionTrue + return res +} + +// DataVolumesReadyCheck returns a check for the "DataVolumesReady" condition. +func DataVolumesReadyCheck(cl client.Client) Checker { + return &dataVolumesReady{ + cl: cl, + } +} diff --git a/internal/health/condition/check_ready_test.go b/internal/health/condition/check_ready_test.go index 03fb65d32..928e9972f 100644 --- a/internal/health/condition/check_ready_test.go +++ b/internal/health/condition/check_ready_test.go @@ -18,7 +18,7 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" + . "github.com/gardener/etcd-druid/internal/health/condition" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/internal/health/etcdmember/builder_test.go b/internal/health/etcdmember/builder_test.go index c0027fd52..6881fab81 100644 --- a/internal/health/etcdmember/builder_test.go +++ b/internal/health/etcdmember/builder_test.go @@ -17,15 +17,15 @@ package etcdmember_test import ( "time" - "k8s.io/utils/pointer" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/gardener/etcd-druid/internal/health/etcdmember" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/etcdmember" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ) var _ = Describe("Builder", func() { diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index b61e6f18b..996184a80 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -19,18 +19,17 @@ import ( "strings" "time" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" ) type readyCheck struct { diff --git a/internal/health/etcdmember/check_ready_test.go b/internal/health/etcdmember/check_ready_test.go index 14e9bfbb9..f36eae29a 100644 --- a/internal/health/etcdmember/check_ready_test.go +++ b/internal/health/etcdmember/check_ready_test.go @@ -20,11 +20,16 @@ import ( "fmt" "time" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/go-logr/logr" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + . "github.com/gardener/etcd-druid/internal/health/etcdmember" + mockclient "github.com/gardener/etcd-druid/internal/mock/controller-runtime/client" + "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/test" + "github.com/go-logr/logr" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,12 +41,6 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - . "github.com/gardener/etcd-druid/pkg/health/etcdmember" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" - "github.com/gardener/etcd-druid/pkg/utils" ) var _ = Describe("ReadyCheck", func() { diff --git a/internal/health/status/check.go b/internal/health/status/check.go index e742f99c1..66ee834b5 100644 --- a/internal/health/status/check.go +++ b/internal/health/status/check.go @@ -25,8 +25,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/health/condition" - "github.com/gardener/etcd-druid/pkg/health/etcdmember" + "github.com/gardener/etcd-druid/internal/health/condition" + "github.com/gardener/etcd-druid/internal/health/etcdmember" ) // ConditionCheckFn is a type alias for a function which returns an implementation of `Check`. @@ -45,9 +45,10 @@ var ( NewDefaultEtcdMemberBuilder = etcdmember.NewBuilder // ConditionChecks Checks are the registered condition checks. ConditionChecks = []ConditionCheckFn{ - condition.AllMembersCheck, + condition.AllMembersReadyCheck, condition.ReadyCheck, condition.BackupReadyCheck, + condition.DataVolumesReadyCheck, } // EtcdMemberChecks are the etcd member checks. EtcdMemberChecks = []EtcdMemberCheckFn{ @@ -85,7 +86,7 @@ func (c *Checker) executeConditionChecks(ctx context.Context, etcd *druidv1alpha wg sync.WaitGroup ) - // Run condition checks in parallel since they work independently from each other. + // Run condition checks in parallel since each check work independently of each other. for _, newCheck := range c.conditionCheckFns { c := newCheck(c.cl) wg.Add(1) diff --git a/internal/health/status/check_test.go b/internal/health/status/check_test.go index f7dee7f65..bba6944db 100644 --- a/internal/health/status/check_test.go +++ b/internal/health/status/check_test.go @@ -18,22 +18,21 @@ import ( "context" "time" - "k8s.io/utils/pointer" - - "github.com/go-logr/logr" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/health/condition" + "github.com/gardener/etcd-druid/internal/health/etcdmember" + . "github.com/gardener/etcd-druid/internal/health/status" - "github.com/gardener/etcd-druid/pkg/health/condition" - "github.com/gardener/etcd-druid/pkg/health/etcdmember" - "github.com/gardener/gardener/pkg/utils/test" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" + + "github.com/gardener/gardener/pkg/utils/test" + "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/status" ) var _ = Describe("Check", func() { @@ -114,6 +113,9 @@ var _ = Describe("Check", func() { func(client.Client) condition.Checker { return createConditionCheck(druidv1alpha1.ConditionTypeBackupReady, druidv1alpha1.ConditionUnknown, "foobar reason", "foobar message") }, + func(client.Client) condition.Checker { + return createConditionCheck(druidv1alpha1.ConditionTypeDataVolumesReady, druidv1alpha1.ConditionUnknown, "foobar reason", "foobar message") + }, })() defer test.WithVar(&EtcdMemberChecks, []EtcdMemberCheckFn{ diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index e07e09778..10d35c0fe 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -16,10 +16,15 @@ package utils import ( "context" + "errors" "fmt" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" @@ -61,3 +66,33 @@ func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.E return nil, nil } + +// FetchPVCWarningEventsForStatefulSet fetches warning events for PVCs for a statefulset and returns them as an error +func FetchPVCWarningEventsForStatefulSet(ctx context.Context, cl client.Client, sts *appsv1.StatefulSet) (string, error) { + pvcs := &corev1.PersistentVolumeClaimList{} + if err := cl.List(ctx, pvcs, client.InNamespace(sts.GetNamespace())); err != nil { + return "", fmt.Errorf("unable to list PVCs for sts %s: %v", sts.Name, err) + } + + var ( + events []string + pvcErr error + ) + + for _, volumeClaim := range sts.Spec.VolumeClaimTemplates { + pvcPrefix := fmt.Sprintf("%s-%s", volumeClaim.Name, sts.Name) + for _, pvc := range pvcs.Items { + if !strings.HasPrefix(pvc.GetName(), pvcPrefix) || pvc.Status.Phase == corev1.ClaimBound { + continue + } + messages, err := kutil.FetchEventMessages(ctx, cl.Scheme(), cl, &pvc, corev1.EventTypeWarning, 2) + if err != nil { + pvcErr = errors.Join(pvcErr, fmt.Errorf("unable to fetch warning events for PVC %s/%s: %v", pvc.Namespace, pvc.Name, err)) + } + if messages != "" { + events = append(events, fmt.Sprintf("Warning for PVC %s/%s: %s", pvc.Namespace, pvc.Name, messages)) + } + } + } + return strings.TrimSpace(strings.Join(events, "; ")), pvcErr +} From dbfe87f248bc3419afcfca51ba124c604e58706e Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 24 Nov 2023 12:25:50 +0530 Subject: [PATCH 011/235] Refactor memberlease Sync() to use OperatorTask for concurrency - Replaced flow package with OperatorTask in Sync method. - Enhanced error handling using multierror for aggregating multiple task errors. --- go.mod | 5 ++-- go.sum | 3 ++ internal/operator/memberlease/memberlease.go | 29 ++++++++++++++------ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 02b12b917..3b4fb5823 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,9 @@ require ( github.com/gardener/gardener v1.86.0 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.4 + github.com/golang/mock v1.6.0 + github.com/google/uuid v1.3.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 github.com/prometheus/client_golang v1.16.0 @@ -73,12 +76,10 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/gophercloud/gophercloud v0.17.0 // indirect github.com/gophercloud/utils v0.0.0-20200204043447-9864b6f1f12f // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 657513f99..df3256986 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -698,6 +700,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 3ca787582..7d626664a 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -7,10 +7,10 @@ import ( "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" + "github.com/hashicorp/go-multierror" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/flow" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,14 +42,27 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKeys := getObjectKeys(etcd) - createFns := make([]flow.TaskFn, len(objectKeys)) - for _, objKey := range objectKeys { - objKey := objKey - createFns = append(createFns, func(ctx context.Context) error { - return r.doCreateOrUpdate(ctx, etcd, objKey) - }) + createTasks := make([]utils.OperatorTask, len(objectKeys)) + var errs error + + for i, objKey := range objectKeys { + objKey := objKey // capture the range variable + createTasks[i] = utils.OperatorTask{ + Name: "CreateOrUpdate-" + objKey.String(), + Fn: func(ctx resource.OperatorContext) error { + return r.doCreateOrUpdate(ctx, etcd, objKey) + }, + } + } + + if errorList := utils.RunConcurrently(ctx, createTasks); len(errorList) > 0 { + for _, err := range errorList { + errs = multierror.Append(errs, err) + } + return errs } - return flow.Parallel(createFns...)(ctx) + + return nil } func (r _resource) doCreateOrUpdate(ctx context.Context, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { From 9bb9cceb43f598ac8d540d20833abc782e772195 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 27 Nov 2023 08:35:14 +0530 Subject: [PATCH 012/235] Refactor: Remove unused etcd field from _resource struct in serviceaccount --- internal/operator/serviceaccount/serviceaccount.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index c436bcdaa..00ee867f9 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -15,7 +15,6 @@ import ( type _resource struct { client client.Client logger logr.Logger - etcd *druidv1alpha1.Etcd disableAutoMount bool } @@ -35,9 +34,9 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { sa := emptyServiceAccount(getObjectKey(etcd)) opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, sa, func() error { - sa.Labels = r.etcd.GetDefaultLabels() - sa.OwnerReferences = []metav1.OwnerReference{r.etcd.GetAsOwnerReference()} - sa.AutomountServiceAccountToken = pointer.Bool(r.disableAutoMount) + sa.Labels = etcd.GetDefaultLabels() + sa.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + sa.AutomountServiceAccountToken = pointer.Bool(!r.disableAutoMount) return nil }) r.logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) From 3ac224439ef3fc5364a198ac2d21f963029416c1 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 27 Nov 2023 08:36:39 +0530 Subject: [PATCH 013/235] Fix: Correct client URL port assignment in createEtcdConfig --- internal/operator/configmap/etcdconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index aec6e21bf..367dd4af4 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -77,7 +77,7 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionMode: utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), AutoCompactionRetention: utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), - ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), + ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), } From 1900d34bfd3b3833cae92069425ec3235c0b5a99 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 27 Nov 2023 13:33:28 +0530 Subject: [PATCH 014/235] Initial support for statefulset --- internal/operator/statefulset/helper.go | 582 +++++++++++++++++++ internal/operator/statefulset/statefulset.go | 454 +++++++++++++++ 2 files changed, 1036 insertions(+) create mode 100644 internal/operator/statefulset/helper.go create mode 100644 internal/operator/statefulset/statefulset.go diff --git a/internal/operator/statefulset/helper.go b/internal/operator/statefulset/helper.go new file mode 100644 index 000000000..980fb48cf --- /dev/null +++ b/internal/operator/statefulset/helper.go @@ -0,0 +1,582 @@ +package statefulset + +import ( + "fmt" + "strconv" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" + corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" +) + +type consistencyLevel string + +const ( + linearizable consistencyLevel = "linearizable" + serializable consistencyLevel = "serializable" + defaultBackupPort int32 = 8080 + defaultServerPort int32 = 2380 + defaultClientPort int32 = 2379 + defaultWrapperPort int32 = 9095 + defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi + defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi + defaultHeartbeatDuration = "10s" + defaultGbcPolicy = "LimitBased" + defaultAutoCompactionRetention = "30m" + defaultEtcdSnapshotTimeout = "15m" + defaultEtcdDefragTimeout = "15m" + defaultAutoCompactionMode = "periodic" + defaultEtcdConnectionTimeout = "5m" +) + +var defaultStorageCapacity = apiresource.MustParse("16Gi") +var defaultResourceRequirements = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse("50m"), + corev1.ResourceMemory: apiresource.MustParse("128Mi"), + }, +} + +func extractObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: etcd.Name, + Namespace: etcd.Namespace, + Labels: etcd.GetDefaultLabels(), + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + } +} +func extractPodTemplateObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Labels: utils.MergeStringMaps(make(map[string]string), etcd.Spec.Labels, etcd.GetDefaultLabels()), + Annotations: etcd.Spec.Annotations, + } +} + +func getEtcdCommandArgs(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []string { + if !useEtcdWrapper { + // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh + return []string{} + } + //TODO @aaronfern: remove this feature gate when UseEtcdWrapper becomes GA + command := []string{"" + "start-etcd"} + command = append(command, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", etcd.Name)) + command = append(command, fmt.Sprintf("--etcd-server-name=%s-local", etcd.Name)) + + clientURLTLS := etcd.Spec.Etcd.ClientUrlTLS + if clientURLTLS == nil { + command = append(command, "--backup-restore-tls-enabled=false") + } else { + dataKey := "ca.crt" + if clientURLTLS.TLSCASecretRef.DataKey != nil { + dataKey = *clientURLTLS.TLSCASecretRef.DataKey + } + command = append(command, "--backup-restore-tls-enabled=true") + command = append(command, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") + command = append(command, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") + command = append(command, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) + } + + return command +} + +func getReadinessHandler(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { + if etcd.Spec.Replicas > 1 { + // TODO(timuthy): Special handling for multi-node etcd can be removed as soon as + // etcd-backup-restore supports `/healthz` for etcd followers, see https://github.com/gardener/etcd-backup-restore/pull/491. + return getReadinessHandlerForMultiNode(useEtcdWrapper, etcd) + } + return getReadinessHandlerForSingleNode(etcd) +} + +func getReadinessHandlerForSingleNode(etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { + scheme := corev1.URISchemeHTTPS + if etcd.Spec.Backup.TLS == nil { + scheme = corev1.URISchemeHTTP + } + + return corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(int(defaultBackupPort)), + Scheme: scheme, + }, + } +} + +func getReadinessHandlerForMultiNode(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { + if useEtcdWrapper { + //TODO @aaronfern: remove this feature gate when UseEtcdWrapper becomes GA + scheme := corev1.URISchemeHTTPS + if etcd.Spec.Backup.TLS == nil { + scheme = corev1.URISchemeHTTP + } + + return corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(int(defaultWrapperPort)), + Scheme: scheme, + }, + } + } + + return corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: getProbeCommand(etcd, linearizable), + }, + } +} + +func getEtcdPorts() []corev1.ContainerPort { + return []corev1.ContainerPort{ + { + Name: "server", + Protocol: "TCP", + ContainerPort: defaultServerPort, + }, + { + Name: "client", + Protocol: "TCP", + ContainerPort: defaultClientPort, + }, + } +} + +func getProbeCommand(etcd *druidv1alpha1.Etcd, consistency consistencyLevel) []string { + var etcdCtlCommand strings.Builder + + etcdCtlCommand.WriteString("ETCDCTL_API=3 etcdctl") + + if etcd.Spec.Etcd.ClientUrlTLS != nil { + dataKey := "ca.crt" + if etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey != nil { + dataKey = *etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey + } + + etcdCtlCommand.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) + etcdCtlCommand.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") + etcdCtlCommand.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") + etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) + + } else { + etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) + } + + etcdCtlCommand.WriteString(" get foo") + + switch consistency { + case linearizable: + etcdCtlCommand.WriteString(" --consistency=l") + case serializable: + etcdCtlCommand.WriteString(" --consistency=s") + } + + return []string{ + "/bin/sh", + "-ec", + etcdCtlCommand.String(), + } +} + +func getEtcdResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { + if etcd.Spec.Etcd.Resources != nil { + return *etcd.Spec.Etcd.Resources + } + + return defaultResourceRequirements +} + +func getEtcdEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { + protocol := "http" + if etcd.Spec.Backup.TLS != nil { + protocol = "https" + } + + endpoint := fmt.Sprintf("%s://%s-local:%d", protocol, etcd.Name, defaultBackupPort) + + return []corev1.EnvVar{ + getEnvVarFromValue("ENABLE_TLS", strconv.FormatBool(etcd.Spec.Backup.TLS != nil)), + getEnvVarFromValue("BACKUP_ENDPOINT", endpoint), + } +} + +func getEnvVarFromValue(name, value string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + Value: value, + } +} + +func getEnvVarFromField(name, fieldPath string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: fieldPath, + }, + }, + } +} + +func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + Key: secretKey, + }, + }, + } +} + +func getEtcdVolumeMounts(etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { + volumeClaimTemplateName := etcd.Name + if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { + volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate + } + + vms := []corev1.VolumeMount{ + { + Name: volumeClaimTemplateName, + MountPath: "/var/etcd/data/", + }, + } + + vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) + + return vms +} + +func getSecretVolumeMounts(clientUrlTLS, peerUrlTLS *druidv1alpha1.TLSConfig) []corev1.VolumeMount { + var vms []corev1.VolumeMount + + if clientUrlTLS != nil { + vms = append(vms, corev1.VolumeMount{ + Name: "client-url-ca-etcd", + MountPath: "/var/etcd/ssl/client/ca", + }, corev1.VolumeMount{ + Name: "client-url-etcd-server-tls", + MountPath: "/var/etcd/ssl/client/server", + }, corev1.VolumeMount{ + Name: "client-url-etcd-client-tls", + MountPath: "/var/etcd/ssl/client/client", + }) + } + + if peerUrlTLS != nil { + vms = append(vms, corev1.VolumeMount{ + Name: "peer-url-ca-etcd", + MountPath: "/var/etcd/ssl/peer/ca", + }, corev1.VolumeMount{ + Name: "peer-url-etcd-server-tls", + MountPath: "/var/etcd/ssl/peer/server", + }) + } + + return vms +} + +func getBackupRestoreCommandArgs(etcd *druidv1alpha1.Etcd) []string { + command := []string{"server"} + + if etcd.Spec.Backup.Store != nil { + command = append(command, "--enable-snapshot-lease-renewal=true") + command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) + command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) + } + + if etcd.Spec.Etcd.DefragmentationSchedule != nil { + command = append(command, "--defragmentation-schedule="+*etcd.Spec.Etcd.DefragmentationSchedule) + } + + if etcd.Spec.Backup.FullSnapshotSchedule != nil { + command = append(command, "--schedule="+*etcd.Spec.Backup.FullSnapshotSchedule) + } + + garbageCollectionPolicy := defaultGbcPolicy + if etcd.Spec.Backup.GarbageCollectionPolicy != nil { + garbageCollectionPolicy = string(*etcd.Spec.Backup.GarbageCollectionPolicy) + } + + command = append(command, "--garbage-collection-policy="+garbageCollectionPolicy) + if garbageCollectionPolicy == "LimitBased" { + command = append(command, "--max-backups=7") + } + + command = append(command, "--data-dir=/var/etcd/data/new.etcd") + command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") + + if etcd.Spec.Backup.Store != nil { + store, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return nil + } + command = append(command, "--storage-provider="+store) + command = append(command, "--store-prefix="+string(etcd.Spec.Backup.Store.Prefix)) + } + + var quota = defaultQuota + if etcd.Spec.Etcd.Quota != nil { + quota = etcd.Spec.Etcd.Quota.Value() + } + + command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) + + if pointer.BoolDeref(etcd.Spec.Backup.EnableProfiling, false) { + command = append(command, "--enable-profiling=true") + } + + if etcd.Spec.Etcd.ClientUrlTLS != nil { + command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") + command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") + command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+pointer.StringDeref(etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt")) + command = append(command, "--insecure-transport=false") + command = append(command, "--insecure-skip-tls-verify=false") + command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) + command = append(command, fmt.Sprintf("--service-endpoints=https://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) + } else { + command = append(command, "--insecure-transport=true") + command = append(command, "--insecure-skip-tls-verify=true") + command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) + command = append(command, fmt.Sprintf("--service-endpoints=http://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) + + } + + if etcd.Spec.Backup.TLS != nil { + command = append(command, "--server-cert=/var/etcd/ssl/client/server/tls.crt") + command = append(command, "--server-key=/var/etcd/ssl/client/server/tls.key") + } + + command = append(command, "--etcd-connection-timeout="+defaultEtcdConnectionTimeout) + + if etcd.Spec.Backup.DeltaSnapshotPeriod != nil { + command = append(command, "--delta-snapshot-period="+etcd.Spec.Backup.DeltaSnapshotPeriod.Duration.String()) + } + + if etcd.Spec.Backup.DeltaSnapshotRetentionPeriod != nil { + command = append(command, "--delta-snapshot-retention-period="+etcd.Spec.Backup.DeltaSnapshotRetentionPeriod.Duration.String()) + } + + var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit + if etcd.Spec.Backup.DeltaSnapshotMemoryLimit != nil { + deltaSnapshotMemoryLimit = etcd.Spec.Backup.DeltaSnapshotMemoryLimit.Value() + } + + command = append(command, "--delta-snapshot-memory-limit="+fmt.Sprint(deltaSnapshotMemoryLimit)) + + if etcd.Spec.Backup.GarbageCollectionPeriod != nil { + command = append(command, "--garbage-collection-period="+etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String()) + } + + if etcd.Spec.Backup.SnapshotCompression != nil { + if pointer.BoolDeref(etcd.Spec.Backup.SnapshotCompression.Enabled, false) { + command = append(command, "--compress-snapshots="+fmt.Sprint(*etcd.Spec.Backup.SnapshotCompression.Enabled)) + } + if etcd.Spec.Backup.SnapshotCompression.Policy != nil { + command = append(command, "--compression-policy="+string(*etcd.Spec.Backup.SnapshotCompression.Policy)) + } + } + + compactionMode := defaultAutoCompactionMode + if etcd.Spec.Common.AutoCompactionMode != nil { + compactionMode = string(*etcd.Spec.Common.AutoCompactionMode) + } + command = append(command, "--auto-compaction-mode="+compactionMode) + + compactionRetention := defaultAutoCompactionRetention + if etcd.Spec.Common.AutoCompactionRetention != nil { + compactionRetention = *etcd.Spec.Common.AutoCompactionRetention + } + command = append(command, "--auto-compaction-retention="+compactionRetention) + + etcdSnapshotTimeout := defaultEtcdSnapshotTimeout + if etcd.Spec.Backup.EtcdSnapshotTimeout != nil { + etcdSnapshotTimeout = etcd.Spec.Backup.EtcdSnapshotTimeout.Duration.String() + } + command = append(command, "--etcd-snapshot-timeout="+etcdSnapshotTimeout) + + etcdDefragTimeout := defaultEtcdDefragTimeout + if etcd.Spec.Etcd.EtcdDefragTimeout != nil { + etcdDefragTimeout = etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String() + } + command = append(command, "--etcd-defrag-timeout="+etcdDefragTimeout) + + command = append(command, "--snapstore-temp-directory=/var/etcd/data/temp") + command = append(command, "--enable-member-lease-renewal=true") + + heartbeatDuration := defaultHeartbeatDuration + if etcd.Spec.Etcd.HeartbeatDuration != nil { + heartbeatDuration = etcd.Spec.Etcd.HeartbeatDuration.Duration.String() + } + command = append(command, "--k8s-heartbeat-duration="+heartbeatDuration) + + if etcd.Spec.Backup.LeaderElection != nil { + if etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout != nil { + command = append(command, "--etcd-connection-timeout-leader-election="+etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout.Duration.String()) + } + + if etcd.Spec.Backup.LeaderElection.ReelectionPeriod != nil { + command = append(command, "--reelection-period="+etcd.Spec.Backup.LeaderElection.ReelectionPeriod.Duration.String()) + } + } + + return command +} + +func getBackupPorts() []corev1.ContainerPort { + return []corev1.ContainerPort{ + { + Name: "server", + Protocol: "TCP", + ContainerPort: defaultBackupPort, + }, + } +} + +func getBackupResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { + if etcd.Spec.Backup.Resources != nil { + return *etcd.Spec.Backup.Resources + } + return defaultResourceRequirements +} + +func getBackupRestoreEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { + var ( + env []corev1.EnvVar + storageContainer string + storeValues = etcd.Spec.Backup.Store + ) + + if etcd.Spec.Backup.Store != nil { + storageContainer = pointer.StringDeref(etcd.Spec.Backup.Store.Container, "") + } + + // TODO(timuthy, shreyas-s-rao): Move STORAGE_CONTAINER a few lines below so that we can append and exit in one step. This should only be done in a release where a restart of etcd is acceptable. + env = append(env, getEnvVarFromValue("STORAGE_CONTAINER", storageContainer)) + env = append(env, getEnvVarFromField("POD_NAME", "metadata.name")) + env = append(env, getEnvVarFromField("POD_NAMESPACE", "metadata.namespace")) + + if storeValues == nil { + return env + } + + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return env + } + + // TODO(timuthy): move this to a non root path when we switch to a rootless distribution + const credentialsMountPath = "/var/etcd-backup" + switch provider { + case utils.S3: + env = append(env, getEnvVarFromValue("AWS_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.ABS: + env = append(env, getEnvVarFromValue("AZURE_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.GCS: + env = append(env, getEnvVarFromValue("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) + + case utils.Swift: + env = append(env, getEnvVarFromValue("OPENSTACK_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.OSS: + env = append(env, getEnvVarFromValue("ALICLOUD_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.ECS: + env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) + env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) + env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) + + case utils.OCS: + env = append(env, getEnvVarFromValue("OPENSHIFT_APPLICATION_CREDENTIALS", credentialsMountPath)) + } + + return env +} + +func getBackupRestoreVolumeMounts(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { + volumeClaimTemplateName := etcd.Name + if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { + volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate + } + vms := []corev1.VolumeMount{ + { + Name: volumeClaimTemplateName, + MountPath: "/var/etcd/data", + }, + { + Name: "etcd-config-file", + MountPath: "/var/etcd/config/", + }, + } + + vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) + + if etcd.Spec.Backup.Store == nil { + return vms + } + + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return vms + } + + switch provider { + case utils.Local: + if etcd.Spec.Backup.Store.Container != nil { + if useEtcdWrapper { + vms = append(vms, corev1.VolumeMount{ + Name: "host-storage", + MountPath: "/home/nonroot/" + pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), + }) + } else { + vms = append(vms, corev1.VolumeMount{ + Name: "host-storage", + MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), + }) + } + } + case utils.GCS: + vms = append(vms, corev1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/.gcp/", + }) + case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + vms = append(vms, corev1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/etcd-backup/", + }) + } + + return vms +} + +func getvolumeClaimTemplateName(etcd *druidv1alpha1.Etcd) string { + volumeClaimTemplateName := etcd.Name + if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { + volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate + } + return volumeClaimTemplateName +} + +func getStorageReq(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { + storageCapacity := defaultStorageCapacity + if etcd.Spec.StorageCapacity != nil { + storageCapacity = *etcd.Spec.StorageCapacity + } + + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageCapacity, + }, + } +} diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go new file mode 100644 index 000000000..b8f5014ad --- /dev/null +++ b/internal/operator/statefulset/statefulset.go @@ -0,0 +1,454 @@ +package statefulset + +import ( + "encoding/json" + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + druidutils "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/gardener/pkg/controllerutils" + gardenerutils "github.com/gardener/gardener/pkg/utils" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type _resource struct { + client client.Client + logger logr.Logger + imageVector imagevector.ImageVector + UseEtcdWrapper bool +} + +func New(client client.Client, logger logr.Logger, config resource.Config) resource.Operator { + + return &_resource{ + client: client, + logger: logger, + imageVector: config.ImageVector, + UseEtcdWrapper: config.UseEtcdWrapper, + } +} + +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) + sts := &appsv1.StatefulSet{} + if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { + if errors.IsNotFound(err) { + return resourceNames, nil + } + return resourceNames, err + } + resourceNames = append(resourceNames, sts.Name) + return resourceNames, nil +} + +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + var ( + existingSts *appsv1.StatefulSet + err error + ) + existingSts, err = r.getExistingStatefulSet(ctx, etcd) + if err != nil { + return err + } + + if existingSts == nil { + return r.createoOrPatch(ctx, etcd, etcd.Spec.Replicas) + } + // Check current TLS status of etcd members + peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) + if err != nil { + return err + } + + // Handling Peer TLS changes + if r.isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { + if err := r.createoOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { + return err + } + tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) + if err != nil { + return fmt.Errorf("error occured while checking IsPeerURLTLSEnabled error :%v", err) + } + if !tlsEnabled { + return fmt.Errorf("TLS not yet enabled for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) + } + + if err := r.deleteAllStsPods(ctx, "deleting all sts pods", existingSts); err != nil { + return err + } + } + + // Check and handle immutable field updates + if existingSts.Generation > 1 && immutableFieldUpdate(existingSts, etcd) { + if err := r.TriggerDelete(ctx, etcd); err != nil { + return err + } + } + + return r.createoOrPatch(ctx, etcd, etcd.Spec.Replicas) +} + +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return client.IgnoreNotFound(r.client.Delete(ctx, emptyStatefulset(getObjectKey(etcd)))) +} + +func (r *_resource) deleteAllStsPods(ctx resource.OperatorContext, opName string, sts *appsv1.StatefulSet) error { + // Get all Pods belonging to the StatefulSet + podList := &corev1.PodList{} + listOpts := []client.ListOption{ + client.InNamespace(sts.Namespace), + client.MatchingLabels(sts.Spec.Selector.MatchLabels), + } + + if err := r.client.List(ctx, podList, listOpts...); err != nil { + r.logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", sts.Name, "Namespace", sts.Namespace) + return err + } + + // Delete each Pod + for _, pod := range podList.Items { + if err := r.client.Delete(ctx, &pod); err != nil { + r.logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) + return err + } + } + + r.logger.Info("All pods in StatefulSet have been deleted", "StatefulSet", sts.Name, "Namespace", sts.Namespace) + return nil +} + +// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled +func (r *_resource) isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { + return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil +} + +func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { + sts := emptyStatefulset(getObjectKey(etcd)) + if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return sts, nil +} + +func (r _resource) getVolumes(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { + vs := []corev1.Volume{ + { + Name: "etcd-config-file", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: etcd.GetConfigMapName(), + }, + Items: []corev1.KeyToPath{ + { + Key: "etcd.conf.yaml", + Path: "etcd.conf.yaml", + }, + }, + DefaultMode: pointer.Int32(0644), + }, + }, + }, + } + + if etcd.Spec.Etcd.ClientUrlTLS != nil { + vs = append(vs, corev1.Volume{ + Name: "client-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "client-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "client-url-etcd-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, + }, + }, + }) + } + + if etcd.Spec.Etcd.PeerUrlTLS != nil { + vs = append(vs, corev1.Volume{ + Name: "peer-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "peer-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, + }, + }, + }) + } + + if etcd.Spec.Backup.Store == nil { + return vs, nil + } + + storeValues := etcd.Spec.Backup.Store + provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + if err != nil { + return vs, nil + } + + switch provider { + case "Local": + hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, r.client, r.logger, storeValues, etcd.GetNamespace()) + if err != nil { + return nil, err + } + + hpt := corev1.HostPathDirectory + vs = append(vs, corev1.Volume{ + Name: "host-storage", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), + Type: &hpt, + }, + }, + }) + case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + if storeValues.SecretRef == nil { + return nil, fmt.Errorf("no secretRef configured for backup store") + } + + vs = append(vs, corev1.Volume{ + Name: "etcd-backup", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: storeValues.SecretRef.Name, + }, + }, + }) + } + + return vs, nil +} + +func (r _resource) createoOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { + desiredStatefulSet := emptyStatefulset(getObjectKey(etcd)) + etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.UseEtcdWrapper) + if err != nil { + return err + } + mutatingFn := func() error { + var stsOriginal = desiredStatefulSet.DeepCopy() + podVolumes, err := r.getVolumes(ctx, etcd) + if err != nil { + return err + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetConfigMapName(), + Namespace: etcd.Namespace, + }, + } + if err := r.client.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { + + return err + } + jsonString, err := json.Marshal(cm.Data) + if err != nil { + return err + } + + configMapChecksum := gardenerutils.ComputeSHA256Hex(jsonString) + + desiredStatefulSet.ObjectMeta = extractObjectMetaFromEtcd(etcd) + desiredStatefulSet.Spec.Replicas = &replicas + desiredStatefulSet.Spec.ServiceName = etcd.GetPeerServiceName() + desiredStatefulSet.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: etcd.GetDefaultLabels(), + } + + desiredStatefulSet.Spec.PodManagementPolicy = appsv1.ParallelPodManagement + desiredStatefulSet.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + } + desiredStatefulSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: getvolumeClaimTemplateName(etcd), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: getStorageReq(etcd), + }, + }, + } + + desiredStatefulSet.Spec.Template.ObjectMeta = extractPodTemplateObjectMetaFromEtcd(etcd) + if len(configMapChecksum) > 0 { + desiredStatefulSet.Spec.Template.Annotations = utils.MergeStringMaps(map[string]string{ + "checksum/etcd-configmap": configMapChecksum, + }, etcd.Spec.Annotations) + } + desiredStatefulSet.Spec.Template.Spec = corev1.PodSpec{ + HostAliases: []corev1.HostAlias{ + { + IP: "127.0.0.1", + Hostnames: []string{etcd.Name + "-local"}, + }, + }, + ServiceAccountName: etcd.GetServiceAccountName(), + Affinity: etcd.Spec.SchedulingConstraints.Affinity, + TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, + Containers: []corev1.Container{ + { + Name: "etcd", + Image: *etcdImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: getEtcdCommandArgs(r.UseEtcdWrapper, etcd), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: getReadinessHandler(r.UseEtcdWrapper, etcd), + InitialDelaySeconds: 15, + PeriodSeconds: 5, + FailureThreshold: 5, + }, + Ports: getEtcdPorts(), + Resources: getEtcdResources(etcd), + Env: getEtcdEnvVars(etcd), + VolumeMounts: getEtcdVolumeMounts(etcd), + }, + { + Name: "backup-restore", + Image: *etcdBackupImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: getBackupRestoreCommandArgs(etcd), + Ports: getBackupPorts(), + Resources: getBackupResources(etcd), + Env: getBackupRestoreEnvVars(etcd), + VolumeMounts: getBackupRestoreVolumeMounts(r.UseEtcdWrapper, etcd), + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "SYS_PTRACE", + }, + }, + }, + }, + }, + ShareProcessNamespace: pointer.Bool(true), + Volumes: podVolumes, + } + + if etcd.Spec.StorageClass != nil && *etcd.Spec.StorageClass != "" { + desiredStatefulSet.Spec.VolumeClaimTemplates[0].Spec.StorageClassName = etcd.Spec.StorageClass + } + if etcd.Spec.PriorityClassName != nil { + desiredStatefulSet.Spec.Template.Spec.PriorityClassName = *etcd.Spec.PriorityClassName + } + if r.UseEtcdWrapper { + // sections to add only when using etcd wrapper + // TODO: @aaronfern add this back to desiredStatefulSet.Spec when UseEtcdWrapper becomes GA + desiredStatefulSet.Spec.Template.Spec.InitContainers = []corev1.Container{ + { + Name: "change-permissions", + Image: *initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{"chown -R 65532:65532 /var/etcd/data"}, + VolumeMounts: getEtcdVolumeMounts(etcd), + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }, + } + if etcd.Spec.Backup.Store != nil { + // Special container to change permissions of backup bucket folder to 65532 (nonroot) + // Only used with local provider + prov, _ := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if prov == utils.Local { + desiredStatefulSet.Spec.Template.Spec.InitContainers = append(desiredStatefulSet.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: "change-backup-bucket-permissions", + Image: *initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, + VolumeMounts: getBackupRestoreVolumeMounts(r.UseEtcdWrapper, etcd), + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }) + } + } + desiredStatefulSet.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ + RunAsGroup: pointer.Int64(65532), + RunAsNonRoot: pointer.Bool(true), + RunAsUser: pointer.Int64(65532), + } + } + + if stsOriginal.Generation > 0 { + // Keep immutable fields + desiredStatefulSet.Spec.PodManagementPolicy = stsOriginal.Spec.PodManagementPolicy + desiredStatefulSet.Spec.ServiceName = stsOriginal.Spec.ServiceName + } + return nil + } + + operationResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) + if err != nil { + return err + } + r.logger.Info("createOrPatch is completed", "namespace", desiredStatefulSet.Namespace, "name", desiredStatefulSet.Name, "operation-result", operationResult) + + return nil +} + +func emptyStatefulset(objectKey client.ObjectKey) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } +} + +func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { + return client.ObjectKey{ + Name: etcd.Name, + Namespace: etcd.Namespace, + } +} + +func immutableFieldUpdate(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { + return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement +} From 315862c0d53d13077ff871ccbbce23244adc7c43 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 27 Nov 2023 13:37:10 +0530 Subject: [PATCH 015/235] Refactor to support statefulset --- internal/controller/etcd/reconcile_spec.go | 53 ++++++++++++---------- internal/controller/etcd/reconciler.go | 29 +++++++++++- internal/operator/registry.go | 52 ++++++++++++++++++--- internal/operator/resource/types.go | 7 +++ 4 files changed, 109 insertions(+), 32 deletions(-) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index bc4910602..96dab25fe 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -1,7 +1,22 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/operator/resource" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" @@ -45,27 +60,17 @@ func (r *Reconciler) recordEtcdSpecReconcileSuspension(etcd *druidv1alpha1.Etcd, ) } -//func (r *Reconciler) reconcileSpec(ctx context.Context, etcd *druidv1alpha1.Etcd, logger logr.Logger) utils.ReconcileStepResult { -// operatorCtx := resource.NewOperatorContext(ctx, r.client, logger, etcd.GetNamespaceName()) -// resourceOperators := r.getOrderedOperatorsForSync() -// for _, operator := range resourceOperators { -// if err := operator.Sync(operatorCtx, etcd); err != nil { -// return utils.ReconcileWithError(err) -// } -// } -// return utils.ContinueReconcile() -//} -// -//func (r *Reconciler) getOrderedOperatorsForSync() []resource.Operator { -// return []resource.Operator{ -// r.operatorRegistry.MemberLeaseOperator(), -// r.operatorRegistry.SnapshotLeaseOperator(), -// r.operatorRegistry.ClientServiceOperator(), -// r.operatorRegistry.PeerServiceOperator(), -// r.operatorRegistry.ConfigMapOperator(), -// r.operatorRegistry.PodDisruptionBudgetOperator(), -// r.operatorRegistry.ServiceAccountOperator(), -// r.operatorRegistry.RoleOperator(), -// r.operatorRegistry.RoleBindingOperator(), -// } -//} +func (r *Reconciler) getOrderedOperatorsForSync() []resource.Operator { + return []resource.Operator{ + r.operatorRegistry.MemberLeaseOperator(), + r.operatorRegistry.SnapshotLeaseOperator(), + r.operatorRegistry.ClientServiceOperator(), + r.operatorRegistry.PeerServiceOperator(), + r.operatorRegistry.ConfigMapOperator(), + r.operatorRegistry.PodDisruptionBudgetOperator(), + r.operatorRegistry.ServiceAccountOperator(), + r.operatorRegistry.RoleOperator(), + r.operatorRegistry.RoleBindingOperator(), + r.operatorRegistry.StatefulSetOperator(), + } +} diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 7229a84e5..eb3cc33ff 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -1,10 +1,26 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package etcd import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -38,8 +54,10 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { } operatorReg := operator.NewRegistry(mgr.GetClient(), logger, - operator.Config{ + resource.Config{ DisableEtcdServiceAccountAutomount: config.DisableEtcdServiceAccountAutomount, + UseEtcdWrapper: config.FeatureGates[features.UseEtcdWrapper], + ImageVector: imageVector, }, ) lastOpErrRecorder := ctrlutils.NewLastOperationErrorRecorder(mgr.GetClient(), logger) @@ -118,7 +136,14 @@ func (r *Reconciler) reconcileSpec(ctx context.Context, etcdObjectKey client.Obj return result } - return ctrlutils.DoNotRequeue() + operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) + resourceOperators := r.getOrderedOperatorsForSync() + for _, operator := range resourceOperators { + if err := operator.Sync(operatorCtx, etcd); err != nil { + return utils.ReconcileWithError(err) + } + } + return utils.ContinueReconcile() } func (r *Reconciler) reconcileStatus(ctx context.Context, etcdNamespacedName types.NamespacedName, runID string) ctrlutils.ReconcileStepResult { diff --git a/internal/operator/registry.go b/internal/operator/registry.go index 3d35b9cae..eb07611e0 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -11,14 +11,11 @@ import ( "github.com/gardener/etcd-druid/internal/operator/rolebinding" "github.com/gardener/etcd-druid/internal/operator/serviceaccount" "github.com/gardener/etcd-druid/internal/operator/snapshotlease" + "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" ) -type Config struct { - DisableEtcdServiceAccountAutomount bool -} - type Registry interface { AllOperators() map[Kind]resource.Operator StatefulSetOperator() resource.Operator @@ -52,7 +49,7 @@ type registry struct { operators map[Kind]resource.Operator } -func NewRegistry(client client.Client, logger logr.Logger, config Config) Registry { +func NewRegistry(client client.Client, logger logr.Logger, config resource.Config) Registry { operators := make(map[Kind]resource.Operator) operators[ConfigMapKind] = configmap.New(client, logger) operators[ServiceAccountKind] = serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount) @@ -63,9 +60,52 @@ func NewRegistry(client client.Client, logger logr.Logger, config Config) Regist operators[PodDisruptionBudgetKind] = poddistruptionbudget.New(client, logger) operators[RoleKind] = role.New(client, logger) operators[RoleBindingKind] = rolebinding.New(client, logger) - return nil + operators[StatefulSetKind] = statefulset.New(client, logger, config) + return registry{ + operators: operators, + } } func (r registry) AllOperators() map[Kind]resource.Operator { return r.operators } + +func (r registry) StatefulSetOperator() resource.Operator { + return r.operators[StatefulSetKind] +} + +func (r registry) ServiceAccountOperator() resource.Operator { + return r.operators[ServiceAccountKind] +} + +func (r registry) RoleOperator() resource.Operator { + return r.operators[RoleKind] +} + +func (r registry) RoleBindingOperator() resource.Operator { + return r.operators[RoleBindingKind] +} + +func (r registry) MemberLeaseOperator() resource.Operator { + return r.operators[MemberLeaseKind] +} + +func (r registry) SnapshotLeaseOperator() resource.Operator { + return r.operators[SnapshotLeaseKind] +} + +func (r registry) ConfigMapOperator() resource.Operator { + return r.operators[ConfigMapKind] +} + +func (r registry) PeerServiceOperator() resource.Operator { + return r.operators[PeerServiceKind] +} + +func (r registry) ClientServiceOperator() resource.Operator { + return r.operators[ClientServiceKind] +} + +func (r registry) PodDisruptionBudgetOperator() resource.Operator { + return r.operators[PodDisruptionBudgetKind] +} diff --git a/internal/operator/resource/types.go b/internal/operator/resource/types.go index 926b47f73..e87641469 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/resource/types.go @@ -4,6 +4,7 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" ) @@ -11,6 +12,12 @@ const ( ConfigMapCheckSumKey = "checksum/etcd-configmap" ) +type Config struct { + DisableEtcdServiceAccountAutomount bool + ImageVector imagevector.ImageVector + UseEtcdWrapper bool +} + type OperatorContext struct { context.Context RunID string From cc61061f1f8eb7aaab1753a9de7baec071986cd4 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 27 Nov 2023 13:39:57 +0530 Subject: [PATCH 016/235] Run etcd-druid locally with new controllers --- internal/controller/compaction/reconciler.go | 30 +++++++++++--------- main.go | 6 ++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 5ccf26755..5c8ce4269 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -20,7 +20,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/utils" druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" @@ -88,7 +87,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result.ReconcileResult() } - return r.reconcileJob(ctx, rLog) + return r.reconcileJob(ctx, rLog, etcd) /* Get latest etcd if it is not found then do not requeue @@ -141,7 +140,8 @@ func (r *Reconciler) reconcileExistingJob(ctx context.Context, logger logr.Logge } func (r *Reconciler) getMatchingJobs(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { - r.client.List(ctx) + // TODO: @seshachalam-yv need to fix + // r.client.List(ctx) } func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { @@ -152,15 +152,16 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd return ctrlutils.ReconcileWithError(err).ReconcileResult() } + // TODO: @seshachalam-yv need to fix // If there is an existing Job then check the Job status and take appropriate action. - if !utils.IsEmptyString(job.Name) { - if result := r.handleExistingJob(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { - return result.ReconcileResult() - } - } - - latestDeltaRevision, err := getLatestDeltaRevision(ctx, etcd.GetDeltaSnapshotLeaseName()) - + // if !utils.IsEmptyString(job.Name) { + // if result := r.handleExistingJob(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { + // return result.ReconcileResult() + // } + // } + + // latestDeltaRevision, err := getLatestDeltaRevision(ctx, etcd.GetDeltaSnapshotLeaseName()) + return ctrlutils.ReconcileWithError(nil).ReconcileResult() } func (r *Reconciler) canScheduleJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { @@ -177,9 +178,12 @@ func (r *Reconciler) getLatestFullSnapshotRevision(ctx context.Context, logger l } func parseRevision(lease *coordinationv1.Lease) (int64, error) { - if lease.Spec.HolderIdentity == nil { + // TODO: @seshachalam-yv need to fix + // if lease.Spec.HolderIdentity == nil { - } + // } + + return int64(0), nil } func (r *Reconciler) handleExistingJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { diff --git a/main.go b/main.go index 37e6d3319..b179f7643 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ import ( "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/gardener/etcd-druid/controllers" + "github.com/gardener/etcd-druid/internal/controller" "github.com/gardener/etcd-druid/pkg/version" flag "github.com/spf13/pflag" @@ -30,7 +30,7 @@ func main() { printVersionInfo() - mgrConfig := controllers.ManagerConfig{} + mgrConfig := controller.ManagerConfig{} if err := mgrConfig.InitFromFlags(flag.CommandLine); err != nil { logger.Error(err, "failed to initialize from flags") os.Exit(1) @@ -45,7 +45,7 @@ func main() { os.Exit(1) } - mgr, err := controllers.CreateManagerWithControllers(&mgrConfig) + mgr, err := controller.CreateManagerWithControllers(&mgrConfig) if err != nil { logger.Error(err, "failed to create druid controller manager") os.Exit(1) From 4f31295eba890f6a2f21ed6a76fb6addc681555f Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 28 Nov 2023 14:20:57 +0530 Subject: [PATCH 017/235] added godocs --- internal/operator/registry.go | 31 ++++++++++++++++++++--------- internal/operator/resource/types.go | 10 ++++++++-- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/internal/operator/registry.go b/internal/operator/registry.go index eb07611e0..f5c33deaf 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -16,7 +16,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// Registry is a facade which gives access to all resource operators. type Registry interface { + // AllOperators gives a map, where the key is the Kind of resource that an operator manages and the value is an Operator itself. AllOperators() map[Kind]resource.Operator StatefulSetOperator() resource.Operator ServiceAccountOperator() resource.Operator @@ -33,15 +35,25 @@ type Registry interface { type Kind string const ( - StatefulSetKind Kind = "StatefulSet" - ServiceAccountKind Kind = "ServiceAccount" - RoleKind Kind = "Role" - RoleBindingKind Kind = "RoleBinding" - MemberLeaseKind Kind = "MemberLease" - SnapshotLeaseKind Kind = "SnapshotLease" - ConfigMapKind Kind = "ConfigMap" - PeerServiceKind Kind = "PeerService" - ClientServiceKind Kind = "ClientService" + // StatefulSetKind indicates that the kind of resource is a StatefulSet. + StatefulSetKind Kind = "StatefulSet" + // ServiceAccountKind indicates that the kind of resource is a ServiceAccount. + ServiceAccountKind Kind = "ServiceAccount" + // RoleKind indicates that the kind of resource is a Role. + RoleKind Kind = "Role" + // RoleBindingKind indicates that the kind of resource is RoleBinding + RoleBindingKind Kind = "RoleBinding" + // MemberLeaseKind indicates that the kind of resource is a Lease used for an etcd member heartbeat. + MemberLeaseKind Kind = "MemberLease" + // SnapshotLeaseKind indicates that the kind of resource is a Lease used to capture snapshot information. + SnapshotLeaseKind Kind = "SnapshotLease" + // ConfigMapKind indicates that the kind of resource is a ConfigMap. + ConfigMapKind Kind = "ConfigMap" + // PeerServiceKind indicates that the kind of resource is a Service used for etcd peer communication. + PeerServiceKind Kind = "PeerService" + // ClientServiceKind indicates that the kind of resource is a Service used for etcd client communication. + ClientServiceKind Kind = "ClientService" + // PodDisruptionBudgetKind indicates that the kind of resource is a PodDisruptionBudget. PodDisruptionBudgetKind Kind = "PodDisruptionBudget" ) @@ -49,6 +61,7 @@ type registry struct { operators map[Kind]resource.Operator } +// NewRegistry creates a new instance of a Registry. func NewRegistry(client client.Client, logger logr.Logger, config resource.Config) Registry { operators := make(map[Kind]resource.Operator) operators[ConfigMapKind] = configmap.New(client, logger) diff --git a/internal/operator/resource/types.go b/internal/operator/resource/types.go index e87641469..7a376f363 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/resource/types.go @@ -9,6 +9,7 @@ import ( ) const ( + // ConfigMapCheckSumKey is the key which stores the latest checksum value of the ConfigMap changes as part of an etcd spec reconciliation. ConfigMapCheckSumKey = "checksum/etcd-configmap" ) @@ -18,13 +19,18 @@ type Config struct { UseEtcdWrapper bool } +// OperatorContext holds the underline context.Context along with additional data that needs to be passed from one reconcile-step to another in a multistep reconciliation run. type OperatorContext struct { context.Context - RunID string + // RunID is unique ID identifying a single reconciliation run. + RunID string + // Logger is the logger that can be used by a reconcile flow or sub-flow. Logger logr.Logger - Data map[string]string + // Data is place-holder for steps to record data that can be accessed by steps ahead in the reconcile flow. + Data map[string]string } +// NewOperatorContext creates a new instance of OperatorContext. func NewOperatorContext(ctx context.Context, logger logr.Logger, runID string) OperatorContext { return OperatorContext{ Context: ctx, From 7a61670fa5d21a1c4e6ad277397ba17ea32e8f9f Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 29 Nov 2023 10:07:28 +0530 Subject: [PATCH 018/235] refactored operator registry and adopted the changes in etcd reconciler --- .../controller/compaction/reconciler_bkp.go | 283 ++++++++++++++++++ internal/controller/etcd/reconciler.go | 35 ++- internal/features/features.go | 2 +- internal/operator/registry.go | 35 +-- internal/operator/resource/types.go | 2 + internal/operator/statefulset/helper.go | 4 +- internal/operator/statefulset/statefulset.go | 43 +-- internal/utils/image_test.go | 2 +- 8 files changed, 349 insertions(+), 57 deletions(-) create mode 100644 internal/controller/compaction/reconciler_bkp.go diff --git a/internal/controller/compaction/reconciler_bkp.go b/internal/controller/compaction/reconciler_bkp.go new file mode 100644 index 000000000..119d102a6 --- /dev/null +++ b/internal/controller/compaction/reconciler_bkp.go @@ -0,0 +1,283 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compaction + +// +//import ( +// "context" +// "time" +// +// druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +// ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" +// druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" +// "github.com/gardener/gardener/pkg/utils/imagevector" +// "github.com/go-logr/logr" +// "github.com/prometheus/client_golang/prometheus" +// batchv1 "k8s.io/api/batch/v1" +// coordinationv1 "k8s.io/api/coordination/v1" +// "k8s.io/apimachinery/pkg/api/errors" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// ctrl "sigs.k8s.io/controller-runtime" +// "sigs.k8s.io/controller-runtime/pkg/client" +// "sigs.k8s.io/controller-runtime/pkg/log" +// "sigs.k8s.io/controller-runtime/pkg/manager" +//) +// +//const ( +// // defaultETCDQuota is the default etcd quota. +// defaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi +//) +// +//// Reconciler reconciles compaction jobs for Etcd resources. +//type Reconciler struct { +// client client.Client +// config *Config +// imageVector imagevector.ImageVector +// logger logr.Logger +//} +// +//// NewReconciler creates a new reconciler for Compaction +//func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { +// imageVector, err := ctrlutils.CreateImageVector() +// if err != nil { +// return nil, err +// } +// return NewReconcilerWithImageVector(mgr, config, imageVector), nil +//} +// +//// NewReconcilerWithImageVector creates a new reconciler for Compaction with an ImageVector. +//func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { +// return &Reconciler{ +// client: mgr.GetClient(), +// config: config, +// imageVector: imageVector, +// logger: log.Log.WithName(controllerName), +// } +//} +// +//// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch +//// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete +//// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;create;list;watch;update;patch;delete +//// +kubebuilder:rbac:groups="",resources=pods,verbs=list;watch;delete;get +// +//// Reconcile reconciles the compaction job. +//func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +// etcd := &druidv1alpha1.Etcd{} +// if result := ctrlutils.GetLatestEtcd(ctx, r.client, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { +// return result.ReconcileResult() +// } +// rLog := r.logger.WithValues("etcd", etcd.GetNamespaceName()) +// jobKey := getJobKey(etcd) +// if etcd.IsMarkedForDeletion() || !etcd.IsBackupEnabled() { +// return r.triggerJobDeletion(ctx, rLog, jobKey) +// } +// +// if result := r.reconcileExistingJob(ctx, rLog, jobKey); ctrlutils.ShortCircuitReconcileFlow(result) { +// return result.ReconcileResult() +// } +// +// return r.reconcileJob(ctx, rLog, etcd) +// /* +// Get latest etcd +// if it is not found then do not requeue +// if there is an error then requeue with error +// +// if it's marked for deletion { +// cleanup any Jobs still out there +// } +// Get Existing Job +// if Job not found then continue +// If error in getting Job requeue with error +// If Job exists and is in deletion, requeue +// If Job is active return with no-requeue +// If Job succeeded record metrics and delete job and continue reconcile +// If Job failed record metrics, delete job and continue reconcile +// Get delta and full leases +// If error requeue with error +// compute difference and if difference > eventsThreshold create Job and record metrics +// */ +// +//} +// +//func getJobKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { +// return client.ObjectKey{ +// Namespace: etcd.Namespace, +// Name: etcd.GetCompactionJobName(), +// } +//} +// +//func (r *Reconciler) reconcileExistingJob(ctx context.Context, logger logr.Logger, jobKey client.ObjectKey) ctrlutils.ReconcileStepResult { +// rjLog := logger.WithValues("operation", "reconcile-compaction-job", "job", jobKey) +// job := &batchv1.Job{} +// if err := r.client.Get(ctx, jobKey, job); !errors.IsNotFound(err) { +// return ctrlutils.ReconcileWithError(err) +// } +// +// type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult +// handlers := []existingJobHandler{ +// r.handleJobDeletionInProgress, +// r.handleActiveJob, +// r.handleSuccessfulJobCompletion, +// r.handleFailedJobCompletion, +// } +// for _, handler := range handlers { +// if result := handler(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { +// return result +// } +// } +// return ctrlutils.ContinueReconcile() +//} +// +//func (r *Reconciler) getMatchingJobs(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { +// // TODO: @seshachalam-yv need to fix +// // r.client.List(ctx) +//} +// +//func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { +// jobObjectKey := client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetCompactionJobName()} +// // Get any existing job that might exist at this time +// job := &batchv1.Job{} +// if err := r.client.Get(ctx, jobObjectKey, job); !errors.IsNotFound(err) { +// return ctrlutils.ReconcileWithError(err).ReconcileResult() +// } +// +// // TODO: @seshachalam-yv need to fix +// // If there is an existing Job then check the Job status and take appropriate action. +// // if !utils.IsEmptyString(job.Name) { +// // if result := r.handleExistingJob(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { +// // return result.ReconcileResult() +// // } +// // } +// +// // latestDeltaRevision, err := getLatestDeltaRevision(ctx, etcd.GetDeltaSnapshotLeaseName()) +// return ctrlutils.ReconcileWithError(nil).ReconcileResult() +//} +// +//func (r *Reconciler) canScheduleJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { +// r.getLatestFullSnapshotRevision(ctx, logger, etcd) +//} +// +//func (r *Reconciler) getLatestFullSnapshotRevision(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (int64, error) { +// fullLease := &coordinationv1.Lease{} +// if err := r.client.Get(ctx, client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetFullSnapshotLeaseName()}, fullLease); err != nil { +// logger.Error(err, "could not fetch full snapshot lease", "lease-name", etcd.GetFullSnapshotLeaseName()) +// return -1, err +// } +// return parseRevision(fullLease) +//} +// +//func parseRevision(lease *coordinationv1.Lease) (int64, error) { +// // TODO: @seshachalam-yv need to fix +// // if lease.Spec.HolderIdentity == nil { +// +// // } +// +// return int64(0), nil +//} +// +//func (r *Reconciler) handleExistingJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { +// type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult +// handlers := []existingJobHandler{ +// r.handleJobDeletionInProgress, +// r.handleActiveJob, +// r.handleSuccessfulJobCompletion, +// r.handleFailedJobCompletion, +// } +// for _, handler := range handlers { +// if result := handler(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { +// return result +// } +// } +// return ctrlutils.ContinueReconcile() +//} +// +//func (r *Reconciler) handleJobDeletionInProgress(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { +// if !job.DeletionTimestamp.IsZero() { +// logger.Info("Deletion has been triggered for the job. A new job can only be created once the previously scheduled job has been deleted") +// return ctrlutils.ReconcileAfter(10*time.Second, "deletion in progress, requeuing job") +// } +// return ctrlutils.ContinueReconcile() +//} +// +//func (r *Reconciler) handleActiveJob(_ context.Context, _ logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { +// metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(1) +// return ctrlutils.DoNotRequeue() +//} +// +//func (r *Reconciler) handleFailedJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { +// if job.Status.Failed > 0 { +// metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) +// if job.Status.StartTime != nil { +// metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: job.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) +// } +// } +// return ctrlutils.ContinueReconcile() +//} +// +//func (r *Reconciler) handleSuccessfulJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { +// if job.Status.Succeeded > 0 { +// metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) +// if job.Status.CompletionTime != nil { +// metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) +// } +// if result := r.deleteJob(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { +// return result +// } +// metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Inc() +// } +// return ctrlutils.ContinueReconcile() +//} +// +//func (r *Reconciler) triggerJobDeletion(ctx context.Context, logger logr.Logger, jobObjectKey client.ObjectKey) (ctrl.Result, error) { +// dLog := logger.WithValues("operation", "delete-compaction-job", "job", jobObjectKey) +// job := &batchv1.Job{} +// if err := r.client.Get(ctx, jobObjectKey, job); err != nil { +// if errors.IsNotFound(err) { +// dLog.Info("No compaction job exists, nothing to clean up") +// return ctrlutils.DoNotRequeue().ReconcileResult() +// } +// return ctrlutils.ReconcileWithError(err).ReconcileResult() +// } +// +// if job.DeletionTimestamp != nil { +// dLog.Info("Deletion has already been triggered for the compaction job. Skipping further action.") +// return ctrlutils.DoNotRequeue().ReconcileResult() +// } +// +// dLog.Info("Triggering delete of compaction job") +// if result := r.deleteJob(ctx, dLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { +// return result.ReconcileResult() +// } +// return ctrlutils.DoNotRequeue().ReconcileResult() +//} +// +//func (r *Reconciler) deleteJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { +// if err := client.IgnoreNotFound(r.client.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { +// logger.Error(err, "error when deleting compaction job") +// return ctrlutils.ReconcileWithError(err) +// } +// return ctrlutils.ContinueReconcile() +//} +// +//func (r *Reconciler) getLatestJob(ctx context.Context, objectKey client.ObjectKey) (*batchv1.Job, error) { +// job := &batchv1.Job{} +// if err := r.client.Get(ctx, objectKey, job); err != nil { +// if errors.IsNotFound(err) { +// return nil, nil +// } +// return nil, err +// } +// return job, nil +//} diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index eb3cc33ff..ee6be61e0 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -20,9 +20,18 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/operator" + "github.com/gardener/etcd-druid/internal/operator/clientservice" + "github.com/gardener/etcd-druid/internal/operator/configmap" + "github.com/gardener/etcd-druid/internal/operator/memberlease" + "github.com/gardener/etcd-druid/internal/operator/peerservice" + "github.com/gardener/etcd-druid/internal/operator/poddistruptionbudget" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/role" + "github.com/gardener/etcd-druid/internal/operator/rolebinding" + "github.com/gardener/etcd-druid/internal/operator/serviceaccount" + "github.com/gardener/etcd-druid/internal/operator/snapshotlease" + "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" "github.com/google/uuid" @@ -52,14 +61,7 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { if err != nil { return nil, err } - operatorReg := operator.NewRegistry(mgr.GetClient(), - logger, - resource.Config{ - DisableEtcdServiceAccountAutomount: config.DisableEtcdServiceAccountAutomount, - UseEtcdWrapper: config.FeatureGates[features.UseEtcdWrapper], - ImageVector: imageVector, - }, - ) + operatorReg := createAndInitializeOperatorRegistry(mgr.GetClient(), logger, config, imageVector) lastOpErrRecorder := ctrlutils.NewLastOperationErrorRecorder(mgr.GetClient(), logger) return &Reconciler{ client: mgr.GetClient(), @@ -72,6 +74,21 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { }, nil } +func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logger, config *Config, imageVector imagevector.ImageVector) operator.Registry { + reg := operator.NewRegistry() + reg.Register(operator.ConfigMapKind, configmap.New(client, logger)) + reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount)) + reg.Register(operator.MemberLeaseKind, memberlease.New(client, logger)) + reg.Register(operator.SnapshotLeaseKind, snapshotlease.New(client, logger)) + reg.Register(operator.ClientServiceKind, clientservice.New(client, logger)) + reg.Register(operator.PeerServiceKind, peerservice.New(client, logger)) + reg.Register(operator.PodDisruptionBudgetKind, poddistruptionbudget.New(client, logger)) + reg.Register(operator.RoleKind, role.New(client, logger)) + reg.Register(operator.RoleBindingKind, rolebinding.New(client, logger)) + reg.Register(operator.StatefulSetKind, statefulset.New(client, logger, imageVector, config.FeatureGates)) + return reg +} + /* If deletionTimestamp set: triggerDeletionFlow(); if err then requeue diff --git a/internal/features/features.go b/internal/features/features.go index a7941de36..b95b36884 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -31,7 +31,7 @@ const ( // changes required for the usage of the etcd-wrapper image. // owner @unmarshall @aaronfern // alpha: v0.19 - UseEtcdWrapper featuregate.Feature = "UseEtcdWrapper" + UseEtcdWrapper featuregate.Feature = "useEtcdWrapper" ) var defaultFeatures = map[featuregate.Feature]featuregate.FeatureSpec{ diff --git a/internal/operator/registry.go b/internal/operator/registry.go index f5c33deaf..af3866e77 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -1,25 +1,16 @@ package operator import ( - "github.com/gardener/etcd-druid/internal/operator/clientservice" - "github.com/gardener/etcd-druid/internal/operator/configmap" - "github.com/gardener/etcd-druid/internal/operator/memberlease" - "github.com/gardener/etcd-druid/internal/operator/peerservice" - "github.com/gardener/etcd-druid/internal/operator/poddistruptionbudget" "github.com/gardener/etcd-druid/internal/operator/resource" - "github.com/gardener/etcd-druid/internal/operator/role" - "github.com/gardener/etcd-druid/internal/operator/rolebinding" - "github.com/gardener/etcd-druid/internal/operator/serviceaccount" - "github.com/gardener/etcd-druid/internal/operator/snapshotlease" - "github.com/gardener/etcd-druid/internal/operator/statefulset" - "github.com/go-logr/logr" - "sigs.k8s.io/controller-runtime/pkg/client" ) // Registry is a facade which gives access to all resource operators. type Registry interface { + // Register provides consumers to register an operator against the kind of resource it operates on. + Register(kind Kind, operator resource.Operator) // AllOperators gives a map, where the key is the Kind of resource that an operator manages and the value is an Operator itself. AllOperators() map[Kind]resource.Operator + GetOperator(kind Kind) resource.Operator StatefulSetOperator() resource.Operator ServiceAccountOperator() resource.Operator RoleOperator() resource.Operator @@ -62,23 +53,21 @@ type registry struct { } // NewRegistry creates a new instance of a Registry. -func NewRegistry(client client.Client, logger logr.Logger, config resource.Config) Registry { +func NewRegistry() Registry { operators := make(map[Kind]resource.Operator) - operators[ConfigMapKind] = configmap.New(client, logger) - operators[ServiceAccountKind] = serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount) - operators[MemberLeaseKind] = memberlease.New(client, logger) - operators[SnapshotLeaseKind] = snapshotlease.New(client, logger) - operators[ClientServiceKind] = clientservice.New(client, logger) - operators[PeerServiceKind] = peerservice.New(client, logger) - operators[PodDisruptionBudgetKind] = poddistruptionbudget.New(client, logger) - operators[RoleKind] = role.New(client, logger) - operators[RoleBindingKind] = rolebinding.New(client, logger) - operators[StatefulSetKind] = statefulset.New(client, logger, config) return registry{ operators: operators, } } +func (r registry) Register(kind Kind, operator resource.Operator) { + r.operators[kind] = operator +} + +func (r registry) GetOperator(kind Kind) resource.Operator { + return r.operators[kind] +} + func (r registry) AllOperators() map[Kind]resource.Operator { return r.operators } diff --git a/internal/operator/resource/types.go b/internal/operator/resource/types.go index 7a376f363..b55f89b4f 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/resource/types.go @@ -51,3 +51,5 @@ type Operator interface { // managed by this Operator then those resource(s) will be either be updated or a deletion is triggered. Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error } + +type OperatorCreatorFn func() Operator diff --git a/internal/operator/statefulset/helper.go b/internal/operator/statefulset/helper.go index 980fb48cf..4a8fd661e 100644 --- a/internal/operator/statefulset/helper.go +++ b/internal/operator/statefulset/helper.go @@ -62,7 +62,7 @@ func getEtcdCommandArgs(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []string // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh return []string{} } - //TODO @aaronfern: remove this feature gate when UseEtcdWrapper becomes GA + //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA command := []string{"" + "start-etcd"} command = append(command, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", etcd.Name)) command = append(command, fmt.Sprintf("--etcd-server-name=%s-local", etcd.Name)) @@ -110,7 +110,7 @@ func getReadinessHandlerForSingleNode(etcd *druidv1alpha1.Etcd) corev1.ProbeHand func getReadinessHandlerForMultiNode(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { if useEtcdWrapper { - //TODO @aaronfern: remove this feature gate when UseEtcdWrapper becomes GA + //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA scheme := corev1.URISchemeHTTPS if etcd.Spec.Backup.TLS == nil { scheme = corev1.URISchemeHTTP diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index b8f5014ad..a8db83082 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -5,6 +5,8 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/features" + "k8s.io/component-base/featuregate" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -25,20 +27,19 @@ type _resource struct { client client.Client logger logr.Logger imageVector imagevector.ImageVector - UseEtcdWrapper bool + useEtcdWrapper bool } -func New(client client.Client, logger logr.Logger, config resource.Config) resource.Operator { - +func New(client client.Client, logger logr.Logger, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) resource.Operator { return &_resource{ client: client, logger: logger, - imageVector: config.ImageVector, - UseEtcdWrapper: config.UseEtcdWrapper, + imageVector: imageVector, + useEtcdWrapper: featureGates[features.UseEtcdWrapper], } } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r *_resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) sts := &appsv1.StatefulSet{} if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { @@ -51,7 +52,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r *_resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { var ( existingSts *appsv1.StatefulSet err error @@ -62,7 +63,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } if existingSts == nil { - return r.createoOrPatch(ctx, etcd, etcd.Spec.Replicas) + return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) } // Check current TLS status of etcd members peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) @@ -72,7 +73,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) // Handling Peer TLS changes if r.isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { - if err := r.createoOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { + if err := r.createOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { return err } tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) @@ -95,10 +96,10 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } } - return r.createoOrPatch(ctx, etcd, etcd.Spec.Replicas) + return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r *_resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { return client.IgnoreNotFound(r.client.Delete(ctx, emptyStatefulset(getObjectKey(etcd)))) } @@ -132,7 +133,7 @@ func (r *_resource) isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bo return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } -func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { +func (r *_resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { sts := emptyStatefulset(getObjectKey(etcd)) if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { if errors.IsNotFound(err) { @@ -143,7 +144,7 @@ func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *dr return sts, nil } -func (r _resource) getVolumes(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { +func (r *_resource) getVolumes(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { vs := []corev1.Volume{ { Name: "etcd-config-file", @@ -255,9 +256,9 @@ func (r _resource) getVolumes(ctx resource.OperatorContext, etcd *druidv1alpha1. return vs, nil } -func (r _resource) createoOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { +func (r *_resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { desiredStatefulSet := emptyStatefulset(getObjectKey(etcd)) - etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.UseEtcdWrapper) + etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.useEtcdWrapper) if err != nil { return err } @@ -331,9 +332,9 @@ func (r _resource) createoOrPatch(ctx resource.OperatorContext, etcd *druidv1alp Name: "etcd", Image: *etcdImage, ImagePullPolicy: corev1.PullIfNotPresent, - Args: getEtcdCommandArgs(r.UseEtcdWrapper, etcd), + Args: getEtcdCommandArgs(r.useEtcdWrapper, etcd), ReadinessProbe: &corev1.Probe{ - ProbeHandler: getReadinessHandler(r.UseEtcdWrapper, etcd), + ProbeHandler: getReadinessHandler(r.useEtcdWrapper, etcd), InitialDelaySeconds: 15, PeriodSeconds: 5, FailureThreshold: 5, @@ -351,7 +352,7 @@ func (r _resource) createoOrPatch(ctx resource.OperatorContext, etcd *druidv1alp Ports: getBackupPorts(), Resources: getBackupResources(etcd), Env: getBackupRestoreEnvVars(etcd), - VolumeMounts: getBackupRestoreVolumeMounts(r.UseEtcdWrapper, etcd), + VolumeMounts: getBackupRestoreVolumeMounts(r.useEtcdWrapper, etcd), SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ Add: []corev1.Capability{ @@ -371,9 +372,9 @@ func (r _resource) createoOrPatch(ctx resource.OperatorContext, etcd *druidv1alp if etcd.Spec.PriorityClassName != nil { desiredStatefulSet.Spec.Template.Spec.PriorityClassName = *etcd.Spec.PriorityClassName } - if r.UseEtcdWrapper { + if r.useEtcdWrapper { // sections to add only when using etcd wrapper - // TODO: @aaronfern add this back to desiredStatefulSet.Spec when UseEtcdWrapper becomes GA + // TODO: @aaronfern add this back to desiredStatefulSet.Spec when useEtcdWrapper becomes GA desiredStatefulSet.Spec.Template.Spec.InitContainers = []corev1.Container{ { Name: "change-permissions", @@ -400,7 +401,7 @@ func (r _resource) createoOrPatch(ctx resource.OperatorContext, etcd *druidv1alp ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, - VolumeMounts: getBackupRestoreVolumeMounts(r.UseEtcdWrapper, etcd), + VolumeMounts: getBackupRestoreVolumeMounts(r.useEtcdWrapper, etcd), SecurityContext: &corev1.SecurityContext{ RunAsGroup: pointer.Int64(0), RunAsNonRoot: pointer.Bool(false), diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index 54734c493..fcc8cc79f 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -102,7 +102,7 @@ var _ = Describe("Image retrieval tests", func() { Expect(initContainerImage).To(BeNil()) }) - It("etcd spec has no images defined, image vector has all images, and UseEtcdWrapper feature gate is turned on", func() { + It("etcd spec has no images defined, image vector has all images, and useEtcdWrapper feature gate is turned on", func() { etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil From 0bb526851e605172c4fdb1f74a918dcdc0f81071 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 29 Nov 2023 11:01:32 +0530 Subject: [PATCH 019/235] removed methods from registry --- internal/operator/registry.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/operator/registry.go b/internal/operator/registry.go index af3866e77..4d2dbf733 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -10,17 +10,9 @@ type Registry interface { Register(kind Kind, operator resource.Operator) // AllOperators gives a map, where the key is the Kind of resource that an operator manages and the value is an Operator itself. AllOperators() map[Kind]resource.Operator + // GetOperator gets an operator that operates on the kind. + // Returns the operator if an operator is found, else nil will be returned. GetOperator(kind Kind) resource.Operator - StatefulSetOperator() resource.Operator - ServiceAccountOperator() resource.Operator - RoleOperator() resource.Operator - RoleBindingOperator() resource.Operator - MemberLeaseOperator() resource.Operator - SnapshotLeaseOperator() resource.Operator - ConfigMapOperator() resource.Operator - PeerServiceOperator() resource.Operator - ClientServiceOperator() resource.Operator - PodDisruptionBudgetOperator() resource.Operator } type Kind string From c9c5772db90a48c10cbe66bbd8647570c7234608 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 29 Nov 2023 11:46:10 +0530 Subject: [PATCH 020/235] cleaned up unused types --- internal/operator/resource/types.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal/operator/resource/types.go b/internal/operator/resource/types.go index b55f89b4f..4038e83e1 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/resource/types.go @@ -4,7 +4,6 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" ) @@ -13,12 +12,6 @@ const ( ConfigMapCheckSumKey = "checksum/etcd-configmap" ) -type Config struct { - DisableEtcdServiceAccountAutomount bool - ImageVector imagevector.ImageVector - UseEtcdWrapper bool -} - // OperatorContext holds the underline context.Context along with additional data that needs to be passed from one reconcile-step to another in a multistep reconciliation run. type OperatorContext struct { context.Context @@ -51,5 +44,3 @@ type Operator interface { // managed by this Operator then those resource(s) will be either be updated or a deletion is triggered. Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error } - -type OperatorCreatorFn func() Operator From 4b34115e19176f9b65eb6b02e17e1bc5ae238561 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Wed, 29 Nov 2023 12:26:47 +0530 Subject: [PATCH 021/235] Include 'internal' directory in Docker build context --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 65603bb91..4b5219563 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,8 @@ !controllers/ !pkg/ !test/ +!internal/ +!vendor/ !.golangci.yaml !go.mod !go.sum From c910d1a067e41c5ed81d627eefdc4363ab4fac25 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Thu, 30 Nov 2023 19:39:53 +0530 Subject: [PATCH 022/235] Refactor statefulSet operator --- internal/controller/etcd/reconcile_spec.go | 26 +- internal/controller/etcd/reconciler.go | 5 +- internal/operator/statefulset/helper.go | 294 ++++++++++++- internal/operator/statefulset/statefulset.go | 426 ++++--------------- 4 files changed, 392 insertions(+), 359 deletions(-) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 96dab25fe..c367f1e82 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -16,7 +16,7 @@ package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" @@ -60,17 +60,17 @@ func (r *Reconciler) recordEtcdSpecReconcileSuspension(etcd *druidv1alpha1.Etcd, ) } -func (r *Reconciler) getOrderedOperatorsForSync() []resource.Operator { - return []resource.Operator{ - r.operatorRegistry.MemberLeaseOperator(), - r.operatorRegistry.SnapshotLeaseOperator(), - r.operatorRegistry.ClientServiceOperator(), - r.operatorRegistry.PeerServiceOperator(), - r.operatorRegistry.ConfigMapOperator(), - r.operatorRegistry.PodDisruptionBudgetOperator(), - r.operatorRegistry.ServiceAccountOperator(), - r.operatorRegistry.RoleOperator(), - r.operatorRegistry.RoleBindingOperator(), - r.operatorRegistry.StatefulSetOperator(), +func (r *Reconciler) getOrderedOperatorsForSync() []operator.Kind { + return []operator.Kind{ + operator.MemberLeaseKind, + operator.SnapshotLeaseKind, + operator.ClientServiceKind, + operator.PeerServiceKind, + operator.ConfigMapKind, + operator.PodDisruptionBudgetKind, + operator.ServiceAccountKind, + operator.RoleKind, + operator.RoleBindingKind, + operator.StatefulSetKind, } } diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index ee6be61e0..5ceeb554f 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -155,8 +155,9 @@ func (r *Reconciler) reconcileSpec(ctx context.Context, etcdObjectKey client.Obj operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) resourceOperators := r.getOrderedOperatorsForSync() - for _, operator := range resourceOperators { - if err := operator.Sync(operatorCtx, etcd); err != nil { + for _, kind := range resourceOperators { + op := r.operatorRegistry.GetOperator(kind) + if err := op.Sync(operatorCtx, etcd); err != nil { return utils.ReconcileWithError(err) } } diff --git a/internal/operator/statefulset/helper.go b/internal/operator/statefulset/helper.go index 4a8fd661e..57753917b 100644 --- a/internal/operator/statefulset/helper.go +++ b/internal/operator/statefulset/helper.go @@ -1,17 +1,23 @@ package statefulset import ( + "encoding/json" "fmt" "strconv" "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" + gardenerutils "github.com/gardener/gardener/pkg/utils" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) type consistencyLevel string @@ -50,10 +56,12 @@ func extractObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, } } -func extractPodTemplateObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { +func extractPodObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd, configMapChecksum string) metav1.ObjectMeta { return metav1.ObjectMeta{ - Labels: utils.MergeStringMaps(make(map[string]string), etcd.Spec.Labels, etcd.GetDefaultLabels()), - Annotations: etcd.Spec.Annotations, + Labels: utils.MergeStringMaps(make(map[string]string), etcd.Spec.Labels, etcd.GetDefaultLabels()), + Annotations: utils.MergeStringMaps(map[string]string{ + resource.ConfigMapCheckSumKey: configMapChecksum, + }, etcd.Spec.Annotations), } } @@ -580,3 +588,283 @@ func getStorageReq(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { }, } } + +func addEtcdContainer(etcdImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { + return &corev1.Container{ + Name: "etcd", + Image: *etcdImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: getEtcdCommandArgs(isEtcdWrapperEnabled, etcd), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: getReadinessHandler(isEtcdWrapperEnabled, etcd), + InitialDelaySeconds: 15, + PeriodSeconds: 5, + FailureThreshold: 5, + }, + Ports: getEtcdPorts(), + Resources: getEtcdResources(etcd), + Env: getEtcdEnvVars(etcd), + VolumeMounts: getEtcdVolumeMounts(etcd), + } +} + +func addBackupRestoreContainer(etcdBackupImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { + return &corev1.Container{ + Name: "backup-restore", + Image: *etcdBackupImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: getBackupRestoreCommandArgs(etcd), + Ports: getBackupPorts(), + Resources: getBackupResources(etcd), + Env: getBackupRestoreEnvVars(etcd), + VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "SYS_PTRACE", + }, + }, + }, + } +} + +func addInitContainersIfWrapperEnabled(initContainerImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) []corev1.Container { + if !isEtcdWrapperEnabled { + return []corev1.Container{} + } + + // Initialize the slice with the 'change-permissions' container + initContainers := []corev1.Container{ + { + Name: "change-permissions", + Image: *initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{"chown -R 65532:65532 /var/etcd/data"}, + VolumeMounts: getEtcdVolumeMounts(etcd), + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }, + } + + if etcd.Spec.Backup.Store != nil { + prov, _ := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if prov == utils.Local { + initContainers = append(initContainers, corev1.Container{ + Name: "change-backup-bucket-permissions", + Image: *initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, + VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }) + } + } + + return initContainers +} + +func getVolumeClaimTemplates(etcd *druidv1alpha1.Etcd) []corev1.PersistentVolumeClaim { + return []corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{ + Name: getvolumeClaimTemplateName(etcd), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: getStorageReq(etcd), + StorageClassName: etcd.Spec.StorageClass, + }, + }, + } +} + +func addSecurityContextIfWrapperEnabled(isEtcdWrapperEnabled bool) *corev1.PodSecurityContext { + return &corev1.PodSecurityContext{ + RunAsGroup: pointer.Int64(65532), + RunAsNonRoot: pointer.Bool(true), + RunAsUser: pointer.Int64(65532), + } +} + +func getPriorityClassName(etcd *druidv1alpha1.Etcd) string { + if etcd.Spec.PriorityClassName != nil { + return *etcd.Spec.PriorityClassName + } + return "" +} + +// hasImmutableFieldChanged checks if any immutable fields have changed in the StatefulSet +// specification compared to the Etcd object. It returns true if there are changes in the immutable fields. +// Currently, it checks for changes in the ServiceName and PodManagementPolicy. +func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { + return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement +} + +func getVolumes(client client.Client, logger logr.Logger, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { + vs := []corev1.Volume{ + { + Name: "etcd-config-file", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: etcd.GetConfigMapName(), + }, + Items: []corev1.KeyToPath{ + { + Key: "etcd.conf.yaml", + Path: "etcd.conf.yaml", + }, + }, + DefaultMode: pointer.Int32(0644), + }, + }, + }, + } + + if etcd.Spec.Etcd.ClientUrlTLS != nil { + vs = append(vs, corev1.Volume{ + Name: "client-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "client-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "client-url-etcd-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, + }, + }, + }) + } + + if etcd.Spec.Etcd.PeerUrlTLS != nil { + vs = append(vs, corev1.Volume{ + Name: "peer-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "peer-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, + }, + }, + }) + } + + if etcd.Spec.Backup.Store == nil { + return vs, nil + } + + storeValues := etcd.Spec.Backup.Store + provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + if err != nil { + return vs, nil + } + + switch provider { + case "Local": + hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, client, logger, storeValues, etcd.GetNamespace()) + if err != nil { + return nil, err + } + + hpt := corev1.HostPathDirectory + vs = append(vs, corev1.Volume{ + Name: "host-storage", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), + Type: &hpt, + }, + }, + }) + case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + if storeValues.SecretRef == nil { + return nil, fmt.Errorf("no secretRef configured for backup store") + } + + vs = append(vs, corev1.Volume{ + Name: "etcd-backup", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: storeValues.SecretRef.Name, + }, + }, + }) + } + + return vs, nil +} + +func getConfigMapChecksum(cl client.Client, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (string, error) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetConfigMapName(), + Namespace: etcd.Namespace, + }, + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { + + return "", err + } + jsonString, err := json.Marshal(cm.Data) + if err != nil { + return "", err + } + + return gardenerutils.ComputeSHA256Hex(jsonString), nil +} + +func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger logr.Logger, opName string, sts *appsv1.StatefulSet) error { + // Get all Pods belonging to the StatefulSet + podList := &corev1.PodList{} + listOpts := []client.ListOption{ + client.InNamespace(sts.Namespace), + client.MatchingLabels(sts.Spec.Selector.MatchLabels), + } + + if err := cl.List(ctx, podList, listOpts...); err != nil { + logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) + return err + } + + for _, pod := range podList.Items { + if err := cl.Delete(ctx, &pod); err != nil { + logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) + return err + } + } + + return nil +} + +// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled +func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { + return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil +} diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index a8db83082..1d231684e 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -1,7 +1,6 @@ package statefulset import ( - "encoding/json" "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -12,7 +11,6 @@ import ( "github.com/gardener/etcd-druid/internal/utils" druidutils "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/controllerutils" - gardenerutils "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -39,101 +37,37 @@ func New(client client.Client, logger logr.Logger, imageVector imagevector.Image } } -func (r *_resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { - resourceNames := make([]string, 0, 1) - sts := &appsv1.StatefulSet{} - if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { - if errors.IsNotFound(err) { - return resourceNames, nil - } - return resourceNames, err - } - resourceNames = append(resourceNames, sts.Name) - return resourceNames, nil +func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + sts, err := r.getExistingStatefulSet(ctx, etcd) + return []string{sts.Name}, err } -func (r *_resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - var ( - existingSts *appsv1.StatefulSet - err error - ) - existingSts, err = r.getExistingStatefulSet(ctx, etcd) +func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + existingSts, err := r.getExistingStatefulSet(ctx, etcd) if err != nil { - return err + return fmt.Errorf("error getting existing StatefulSet: %w", err) } if existingSts == nil { - return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) - } - // Check current TLS status of etcd members - peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) - if err != nil { - return err + return r.doCreateOrPatch(ctx, etcd, etcd.Spec.Replicas) } - // Handling Peer TLS changes - if r.isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { - if err := r.createOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { - return err - } - tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) - if err != nil { - return fmt.Errorf("error occured while checking IsPeerURLTLSEnabled error :%v", err) - } - if !tlsEnabled { - return fmt.Errorf("TLS not yet enabled for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) - } - - if err := r.deleteAllStsPods(ctx, "deleting all sts pods", existingSts); err != nil { - return err - } + if err := r.handlePeerTLSEnabled(ctx, etcd, existingSts); err != nil { + return fmt.Errorf("error handling peer TLS: %w", err) } - // Check and handle immutable field updates - if existingSts.Generation > 1 && immutableFieldUpdate(existingSts, etcd) { - if err := r.TriggerDelete(ctx, etcd); err != nil { - return err - } + if err := r.handleImmutableFieldUpdates(ctx, etcd, existingSts); err != nil { + return fmt.Errorf("error handling immutable field updates: %w", err) } - return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) + return r.doCreateOrPatch(ctx, etcd, etcd.Spec.Replicas) } -func (r *_resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { return client.IgnoreNotFound(r.client.Delete(ctx, emptyStatefulset(getObjectKey(etcd)))) } -func (r *_resource) deleteAllStsPods(ctx resource.OperatorContext, opName string, sts *appsv1.StatefulSet) error { - // Get all Pods belonging to the StatefulSet - podList := &corev1.PodList{} - listOpts := []client.ListOption{ - client.InNamespace(sts.Namespace), - client.MatchingLabels(sts.Spec.Selector.MatchLabels), - } - - if err := r.client.List(ctx, podList, listOpts...); err != nil { - r.logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", sts.Name, "Namespace", sts.Namespace) - return err - } - - // Delete each Pod - for _, pod := range podList.Items { - if err := r.client.Delete(ctx, &pod); err != nil { - r.logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) - return err - } - } - - r.logger.Info("All pods in StatefulSet have been deleted", "StatefulSet", sts.Name, "Namespace", sts.Namespace) - return nil -} - -// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled -func (r *_resource) isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { - return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil -} - -func (r *_resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { +func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { sts := emptyStatefulset(getObjectKey(etcd)) if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { if errors.IsNotFound(err) { @@ -144,292 +78,106 @@ func (r *_resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *d return sts, nil } -func (r *_resource) getVolumes(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { - vs := []corev1.Volume{ - { - Name: "etcd-config-file", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: etcd.GetConfigMapName(), - }, - Items: []corev1.KeyToPath{ - { - Key: "etcd.conf.yaml", - Path: "etcd.conf.yaml", - }, - }, - DefaultMode: pointer.Int32(0644), - }, - }, - }, +func (r _resource) doCreateOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { + desiredStatefulSet := emptyStatefulset(getObjectKey(etcd)) + etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.useEtcdWrapper) + if err != nil { + return err + } + podVolumes, err := getVolumes(r.client, r.logger, ctx, etcd) + if err != nil { + return err } - if etcd.Spec.Etcd.ClientUrlTLS != nil { - vs = append(vs, corev1.Volume{ - Name: "client-url-ca-etcd", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "client-url-etcd-server-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "client-url-etcd-client-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - }, - }, - }) + configMapChecksum, err := getConfigMapChecksum(r.client, ctx, etcd) + if err != nil { + return err } - if etcd.Spec.Etcd.PeerUrlTLS != nil { - vs = append(vs, corev1.Volume{ - Name: "peer-url-ca-etcd", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, - }, + mutatingFn := func() error { + desiredStatefulSet.ObjectMeta = extractObjectMetaFromEtcd(etcd) + desiredStatefulSet.Spec = appsv1.StatefulSetSpec{ + Replicas: &replicas, + ServiceName: etcd.GetPeerServiceName(), + Selector: &metav1.LabelSelector{ + MatchLabels: etcd.GetDefaultLabels(), }, - }, - corev1.Volume{ - Name: "peer-url-etcd-server-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, + PodManagementPolicy: appsv1.ParallelPodManagement, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + VolumeClaimTemplates: getVolumeClaimTemplates(etcd), + Template: corev1.PodTemplateSpec{ + ObjectMeta: extractPodObjectMetaFromEtcd(etcd, configMapChecksum), + Spec: corev1.PodSpec{ + HostAliases: []corev1.HostAlias{{ + IP: "127.0.0.1", + Hostnames: []string{etcd.Name + "-local"}}}, + ServiceAccountName: etcd.GetServiceAccountName(), + Affinity: etcd.Spec.SchedulingConstraints.Affinity, + TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, + Containers: []corev1.Container{ + *addEtcdContainer(etcdImage, r.useEtcdWrapper, etcd), + *addBackupRestoreContainer(etcdBackupImage, r.useEtcdWrapper, etcd), }, + InitContainers: addInitContainersIfWrapperEnabled(initContainerImage, r.useEtcdWrapper, etcd), + ShareProcessNamespace: pointer.Bool(true), + Volumes: podVolumes, + SecurityContext: addSecurityContextIfWrapperEnabled(r.useEtcdWrapper), + PriorityClassName: getPriorityClassName(etcd), }, - }) - } - - if etcd.Spec.Backup.Store == nil { - return vs, nil + }, + } + return nil } - storeValues := etcd.Spec.Backup.Store - provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) if err != nil { - return vs, nil + return err } - switch provider { - case "Local": - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, r.client, r.logger, storeValues, etcd.GetNamespace()) - if err != nil { - return nil, err - } + r.logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) + return nil +} - hpt := corev1.HostPathDirectory - vs = append(vs, corev1.Volume{ - Name: "host-storage", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), - Type: &hpt, - }, - }, - }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: - if storeValues.SecretRef == nil { - return nil, fmt.Errorf("no secretRef configured for backup store") - } +func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { + if existingSts.Generation > 1 && hasImmutableFieldChanged(existingSts, etcd) { + r.logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") - vs = append(vs, corev1.Volume{ - Name: "etcd-backup", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, - }, - }, - }) + if err := r.TriggerDelete(ctx, etcd); err != nil { + return fmt.Errorf("error deleting StatefulSet with immutable field updates: %v", err) + } } - - return vs, nil + return nil } -func (r *_resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { - desiredStatefulSet := emptyStatefulset(getObjectKey(etcd)) - etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.useEtcdWrapper) +func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { + peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) if err != nil { - return err + return fmt.Errorf("error checking if peer URL TLS is enabled: %v", err) } - mutatingFn := func() error { - var stsOriginal = desiredStatefulSet.DeepCopy() - podVolumes, err := r.getVolumes(ctx, etcd) - if err != nil { - return err - } - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetConfigMapName(), - Namespace: etcd.Namespace, - }, - } - if err := r.client.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { + if isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { + r.logger.Info("Enabling TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) - return err - } - jsonString, err := json.Marshal(cm.Data) - if err != nil { - return err + if err := r.doCreateOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { + return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %v", err) } - configMapChecksum := gardenerutils.ComputeSHA256Hex(jsonString) - - desiredStatefulSet.ObjectMeta = extractObjectMetaFromEtcd(etcd) - desiredStatefulSet.Spec.Replicas = &replicas - desiredStatefulSet.Spec.ServiceName = etcd.GetPeerServiceName() - desiredStatefulSet.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: etcd.GetDefaultLabels(), - } - - desiredStatefulSet.Spec.PodManagementPolicy = appsv1.ParallelPodManagement - desiredStatefulSet.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ - Type: appsv1.RollingUpdateStatefulSetStrategyType, - } - desiredStatefulSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: getvolumeClaimTemplateName(etcd), - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: getStorageReq(etcd), - }, - }, - } - - desiredStatefulSet.Spec.Template.ObjectMeta = extractPodTemplateObjectMetaFromEtcd(etcd) - if len(configMapChecksum) > 0 { - desiredStatefulSet.Spec.Template.Annotations = utils.MergeStringMaps(map[string]string{ - "checksum/etcd-configmap": configMapChecksum, - }, etcd.Spec.Annotations) - } - desiredStatefulSet.Spec.Template.Spec = corev1.PodSpec{ - HostAliases: []corev1.HostAlias{ - { - IP: "127.0.0.1", - Hostnames: []string{etcd.Name + "-local"}, - }, - }, - ServiceAccountName: etcd.GetServiceAccountName(), - Affinity: etcd.Spec.SchedulingConstraints.Affinity, - TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, - Containers: []corev1.Container{ - { - Name: "etcd", - Image: *etcdImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: getEtcdCommandArgs(r.useEtcdWrapper, etcd), - ReadinessProbe: &corev1.Probe{ - ProbeHandler: getReadinessHandler(r.useEtcdWrapper, etcd), - InitialDelaySeconds: 15, - PeriodSeconds: 5, - FailureThreshold: 5, - }, - Ports: getEtcdPorts(), - Resources: getEtcdResources(etcd), - Env: getEtcdEnvVars(etcd), - VolumeMounts: getEtcdVolumeMounts(etcd), - }, - { - Name: "backup-restore", - Image: *etcdBackupImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: getBackupRestoreCommandArgs(etcd), - Ports: getBackupPorts(), - Resources: getBackupResources(etcd), - Env: getBackupRestoreEnvVars(etcd), - VolumeMounts: getBackupRestoreVolumeMounts(r.useEtcdWrapper, etcd), - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_PTRACE", - }, - }, - }, - }, - }, - ShareProcessNamespace: pointer.Bool(true), - Volumes: podVolumes, - } - - if etcd.Spec.StorageClass != nil && *etcd.Spec.StorageClass != "" { - desiredStatefulSet.Spec.VolumeClaimTemplates[0].Spec.StorageClassName = etcd.Spec.StorageClass - } - if etcd.Spec.PriorityClassName != nil { - desiredStatefulSet.Spec.Template.Spec.PriorityClassName = *etcd.Spec.PriorityClassName + tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) + if err != nil { + return fmt.Errorf("error verifying if TLS is enabled post-patch: %v", err) } - if r.useEtcdWrapper { - // sections to add only when using etcd wrapper - // TODO: @aaronfern add this back to desiredStatefulSet.Spec when useEtcdWrapper becomes GA - desiredStatefulSet.Spec.Template.Spec.InitContainers = []corev1.Container{ - { - Name: "change-permissions", - Image: *initContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{"chown -R 65532:65532 /var/etcd/data"}, - VolumeMounts: getEtcdVolumeMounts(etcd), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }, - } - if etcd.Spec.Backup.Store != nil { - // Special container to change permissions of backup bucket folder to 65532 (nonroot) - // Only used with local provider - prov, _ := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if prov == utils.Local { - desiredStatefulSet.Spec.Template.Spec.InitContainers = append(desiredStatefulSet.Spec.Template.Spec.InitContainers, corev1.Container{ - Name: "change-backup-bucket-permissions", - Image: *initContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, - VolumeMounts: getBackupRestoreVolumeMounts(r.useEtcdWrapper, etcd), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }) - } - } - desiredStatefulSet.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ - RunAsGroup: pointer.Int64(65532), - RunAsNonRoot: pointer.Bool(true), - RunAsUser: pointer.Int64(65532), - } + if !tlsEnabled { + return fmt.Errorf("failed to enable TLS for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) } - if stsOriginal.Generation > 0 { - // Keep immutable fields - desiredStatefulSet.Spec.PodManagementPolicy = stsOriginal.Spec.PodManagementPolicy - desiredStatefulSet.Spec.ServiceName = stsOriginal.Spec.ServiceName + if err := deleteAllStsPods(ctx, r.client, r.logger, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { + return fmt.Errorf("error deleting StatefulSet pods after enabling TLS: %v", err) } - return nil - } - operationResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) - if err != nil { - return err + r.logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) } - r.logger.Info("createOrPatch is completed", "namespace", desiredStatefulSet.Namespace, "name", desiredStatefulSet.Name, "operation-result", operationResult) return nil } @@ -449,7 +197,3 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { Namespace: etcd.Namespace, } } - -func immutableFieldUpdate(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { - return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement -} From 7419b885c401292b3df790235344ac9542ffcba7 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 4 Dec 2023 15:53:06 +0530 Subject: [PATCH 023/235] WIP commit - refactored sts resource operator --- api/v1alpha1/types_etcd.go | 4 +- internal/controller/compaction/reconciler.go | 2 +- .../controller/compaction/reconciler_bkp.go | 2 +- internal/features/features.go | 2 +- .../serviceaccount/serviceaccount_test.go | 12 + .../operator/snapshotlease/snapshotlease.go | 2 +- internal/operator/statefulset/builder.go | 751 +++++++ internal/operator/statefulset/helper.go | 1736 ++++++++--------- internal/operator/statefulset/statefulset.go | 169 +- internal/utils/image.go | 10 +- internal/utils/miscellaneous.go | 16 + 11 files changed, 1768 insertions(+), 938 deletions(-) create mode 100644 internal/operator/serviceaccount/serviceaccount_test.go create mode 100644 internal/operator/statefulset/builder.go diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index c84db86a2..d9fefa478 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -567,8 +567,8 @@ func (e *Etcd) GetRoleBindingName() string { return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, e.Name) } -// IsBackupEnabled returns true if backup has been enabled for this etcd, else returns false. -func (e *Etcd) IsBackupEnabled() bool { +// IsBackupStoreEnabled returns true if backup store has been enabled for this etcd, else returns false. +func (e *Etcd) IsBackupStoreEnabled() bool { return e.Spec.Backup.Store != nil } diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 5c8ce4269..3333d3697 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -79,7 +79,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } rLog := r.logger.WithValues("etcd", etcd.GetNamespaceName()) jobKey := getJobKey(etcd) - if etcd.IsMarkedForDeletion() || !etcd.IsBackupEnabled() { + if etcd.IsMarkedForDeletion() || !etcd.IsBackupStoreEnabled() { return r.triggerJobDeletion(ctx, rLog, jobKey) } diff --git a/internal/controller/compaction/reconciler_bkp.go b/internal/controller/compaction/reconciler_bkp.go index 119d102a6..85300dcdf 100644 --- a/internal/controller/compaction/reconciler_bkp.go +++ b/internal/controller/compaction/reconciler_bkp.go @@ -80,7 +80,7 @@ package compaction // } // rLog := r.logger.WithValues("etcd", etcd.GetNamespaceName()) // jobKey := getJobKey(etcd) -// if etcd.IsMarkedForDeletion() || !etcd.IsBackupEnabled() { +// if etcd.IsMarkedForDeletion() || !etcd.IsBackupStoreEnabled() { // return r.triggerJobDeletion(ctx, rLog, jobKey) // } // diff --git a/internal/features/features.go b/internal/features/features.go index b95b36884..a7941de36 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -31,7 +31,7 @@ const ( // changes required for the usage of the etcd-wrapper image. // owner @unmarshall @aaronfern // alpha: v0.19 - UseEtcdWrapper featuregate.Feature = "useEtcdWrapper" + UseEtcdWrapper featuregate.Feature = "UseEtcdWrapper" ) var defaultFeatures = map[featuregate.Feature]featuregate.FeatureSpec{ diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go new file mode 100644 index 000000000..de32bbbf3 --- /dev/null +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -0,0 +1,12 @@ +package serviceaccount + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func DummyTest(t *testing.T) { + g := NewWithT(t) + g.Expect("bingo").To(Equal("bingo")) +} diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 6d5d414ed..100f139a8 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -57,7 +57,7 @@ func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*c } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - if !etcd.IsBackupEnabled() { + if !etcd.IsBackupStoreEnabled() { r.logger.Info("Backup has been disabled. Triggering delete of snapshot leases") return r.TriggerDelete(ctx, etcd) } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go new file mode 100644 index 000000000..52780b26b --- /dev/null +++ b/internal/operator/statefulset/builder.go @@ -0,0 +1,751 @@ +package statefulset + +import ( + "fmt" + "strconv" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + druidutils "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// defaults +// ----------------------------------------------------------------------------------------- +const ( + defaultBackupPort int32 = 8080 + defaultServerPort int32 = 2380 + defaultClientPort int32 = 2379 + defaultWrapperPort int = 9095 + defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi + defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi + defaultHeartbeatDuration = "10s" + defaultGbcPolicy = "LimitBased" + defaultAutoCompactionRetention = "30m" + defaultEtcdSnapshotTimeout = "15m" + defaultEtcdDefragTimeout = "15m" + defaultAutoCompactionMode = "periodic" + defaultEtcdConnectionTimeout = "5m" + defaultPodManagementPolicy = appsv1.ParallelPodManagement +) + +var ( + defaultStorageCapacity = apiresource.MustParse("16Gi") + defaultResourceRequirements = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse("50m"), + corev1.ResourceMemory: apiresource.MustParse("128Mi"), + }, + } + defaultUpdateStrategy = appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType} +) + +type stsBuilder struct { + client client.Client + etcd *druidv1alpha1.Etcd + replicas int32 + useEtcdWrapper bool + provider *string + etcdImage string + etcdBackupRestoreImage string + initContainerImage string + sts *appsv1.StatefulSet + logger logr.Logger +} + +func newStsBuilder(client client.Client, + logger logr.Logger, + etcd *druidv1alpha1.Etcd, + replicas int32, + useEtcdWrapper bool, + imageVector imagevector.ImageVector, + sts *appsv1.StatefulSet) (*stsBuilder, error) { + etcdImage, etcdBackupRestoreImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, imageVector, useEtcdWrapper) + if err != nil { + return nil, err + } + provider, err := getBackupStoreProvider(etcd) + if err != nil { + return nil, err + } + return &stsBuilder{ + client: client, + logger: logger, + etcd: etcd, + replicas: replicas, + useEtcdWrapper: useEtcdWrapper, + provider: provider, + etcdImage: etcdImage, + etcdBackupRestoreImage: etcdBackupRestoreImage, + initContainerImage: initContainerImage, + sts: sts, + }, nil +} + +func (b *stsBuilder) Build(ctx resource.OperatorContext) error { + b.createStatefulSetObjectMeta() + if err := b.createStatefulSetSpec(ctx); err != nil { + return err + } + return nil +} + +func (b *stsBuilder) createStatefulSetObjectMeta() { + b.sts.ObjectMeta = metav1.ObjectMeta{ + Name: b.etcd.Name, + Namespace: b.etcd.Namespace, + Labels: b.etcd.GetDefaultLabels(), + OwnerReferences: []metav1.OwnerReference{b.etcd.GetAsOwnerReference()}, + } +} + +func (b *stsBuilder) createStatefulSetSpec(ctx resource.OperatorContext) error { + podVolumes, err := b.getPodVolumes(ctx) + if err != nil { + return err + } + b.sts.Spec = appsv1.StatefulSetSpec{ + Replicas: pointer.Int32(b.replicas), + Selector: &metav1.LabelSelector{ + MatchLabels: b.etcd.GetDefaultLabels(), + }, + PodManagementPolicy: defaultPodManagementPolicy, + UpdateStrategy: defaultUpdateStrategy, + VolumeClaimTemplates: b.getVolumeClaimTemplates(), + ServiceName: b.etcd.GetPeerServiceName(), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: utils.MergeMaps[string](b.etcd.Spec.Labels, b.etcd.GetDefaultLabels()), + Annotations: b.getPodTemplateAnnotations(ctx), + }, + Spec: corev1.PodSpec{ + HostAliases: b.getHostAliases(), + ServiceAccountName: b.etcd.GetServiceAccountName(), + ShareProcessNamespace: pointer.Bool(true), + InitContainers: b.getPodInitContainers(), + Containers: []corev1.Container{ + b.getEtcdContainer(), + b.getBackupRestoreContainer(), + }, + SecurityContext: b.getPodSecurityContext(), + Affinity: b.etcd.Spec.SchedulingConstraints.Affinity, + TopologySpreadConstraints: b.etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, + Volumes: podVolumes, + PriorityClassName: utils.TypeDeref[string](b.etcd.Spec.PriorityClassName, ""), + }, + }, + } + return nil +} + +func (b *stsBuilder) getHostAliases() []corev1.HostAlias { + return []corev1.HostAlias{ + { + IP: "127.0.0.1", + Hostnames: []string{b.etcd.Name + "-local"}, + }, + } +} + +func (b *stsBuilder) getPodTemplateAnnotations(ctx resource.OperatorContext) map[string]string { + if configMapCheckSum, ok := ctx.Data[resource.ConfigMapCheckSumKey]; ok { + return utils.MergeMaps[string](b.etcd.Spec.Annotations, map[string]string{ + resource.ConfigMapCheckSumKey: configMapCheckSum, + }) + } + return b.etcd.Spec.Annotations +} + +func (b *stsBuilder) getVolumeClaimTemplates() []corev1.PersistentVolumeClaim { + return []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: utils.TypeDeref[apiresource.Quantity](b.etcd.Spec.StorageCapacity, defaultStorageCapacity), + }, + }, + StorageClassName: b.etcd.Spec.StorageClass, + }, + }, + } +} + +func (b *stsBuilder) getPodInitContainers() []corev1.Container { + initContainers := make([]corev1.Container, 0, 2) + if !b.useEtcdWrapper { + return initContainers + } + initContainers = append(initContainers, corev1.Container{ + Name: "change-permissions", + Image: b.initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{"chown -R 65532:65532 /var/etcd/data"}, + VolumeMounts: []corev1.VolumeMount{b.getEtcdDataVolumeMount()}, + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }) + if b.etcd.IsBackupStoreEnabled() { + if b.provider != nil && *b.provider == utils.Local { + initContainers = append(initContainers, corev1.Container{ + Name: "change-backup-bucket-permissions", + Image: b.initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *b.etcd.Spec.Backup.Store.Container)}, + VolumeMounts: b.getBackupRestoreContainerVolumeMounts(), + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }) + } + } + return initContainers +} + +func (b *stsBuilder) getEtcdContainerVolumeMounts() []corev1.VolumeMount { + etcdVolumeMounts := make([]corev1.VolumeMount, 0, 6) + etcdVolumeMounts = append(etcdVolumeMounts, b.getEtcdDataVolumeMount()) + etcdVolumeMounts = append(etcdVolumeMounts, b.getSecretVolumeMounts()...) + return etcdVolumeMounts +} + +func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMount { + brVolumeMounts := make([]corev1.VolumeMount, 0, 8) + + brVolumeMounts = append(brVolumeMounts, + b.getEtcdDataVolumeMount(), + corev1.VolumeMount{ + Name: "etcd-config-file", + MountPath: "/var/etcd/config/", + }, + ) + brVolumeMounts = append(brVolumeMounts, b.getSecretVolumeMounts()...) + + if b.etcd.IsBackupStoreEnabled() { + etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() + if etcdBackupVolumeMount != nil { + brVolumeMounts = append(brVolumeMounts, *b.getEtcdBackupVolumeMount()) + } + } + + return brVolumeMounts +} + +func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { + switch *b.provider { + case utils.Local: + if b.etcd.Spec.Backup.Store.Container != nil { + if b.useEtcdWrapper { + return &corev1.VolumeMount{ + Name: "host-storage", + MountPath: "/home/nonroot/" + pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, ""), + } + } else { + return &corev1.VolumeMount{ + Name: "host-storage", + MountPath: pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, ""), + } + } + } + case utils.GCS: + return &corev1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/.gcp/", + } + case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + return &corev1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/etcd-backup/", + } + } + return nil +} + +func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { + volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) + return corev1.VolumeMount{ + Name: volumeClaimTemplateName, + MountPath: "/var/etcd/data/", + } +} + +func (b *stsBuilder) getEtcdContainer() corev1.Container { + return corev1.Container{ + Name: "etcd", + Image: b.etcdImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: b.getEtcdContainerCommandArgs(), + ReadinessProbe: b.getEtcdContainerReadinessProbe(), + Ports: []corev1.ContainerPort{ + { + Name: "server", + Protocol: "TCP", + ContainerPort: defaultServerPort, + }, + { + Name: "client", + Protocol: "TCP", + ContainerPort: defaultClientPort, + }, + }, + Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Etcd.Resources, defaultResourceRequirements), + Env: b.getEtcdContainerEnvVars(), + VolumeMounts: b.getEtcdContainerVolumeMounts(), + } +} + +func (b *stsBuilder) getBackupRestoreContainer() corev1.Container { + return corev1.Container{ + Name: "backup-restore", + Image: b.etcdBackupRestoreImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: nil, + Ports: nil, + Env: nil, + Resources: corev1.ResourceRequirements{}, + VolumeMounts: nil, + } + +} + +func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { + commandArgs := []string{"server"} + + // Backup store related command line args + // ----------------------------------------------------------------------------------------------------------------- + if b.etcd.IsBackupStoreEnabled() { + commandArgs = append(commandArgs, b.getBackupStoreCommandArgs()...) + } + + // Defragmentation command line args + // ----------------------------------------------------------------------------------------------------------------- + if b.etcd.Spec.Etcd.DefragmentationSchedule != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--defragmentation-schedule=%s", *b.etcd.Spec.Etcd.DefragmentationSchedule)) + } + etcdDefragTimeout := defaultEtcdDefragTimeout + if b.etcd.Spec.Etcd.EtcdDefragTimeout != nil { + etcdDefragTimeout = b.etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String() + } + commandArgs = append(commandArgs, "--etcd-defrag-timeout="+etcdDefragTimeout) + + // Compaction command line args + // ----------------------------------------------------------------------------------------------------------------- + compactionMode := defaultAutoCompactionMode + if b.etcd.Spec.Common.AutoCompactionMode != nil { + compactionMode = string(*b.etcd.Spec.Common.AutoCompactionMode) + } + commandArgs = append(commandArgs, "--auto-compaction-mode="+compactionMode) + + compactionRetention := defaultAutoCompactionRetention + if b.etcd.Spec.Common.AutoCompactionRetention != nil { + compactionRetention = *b.etcd.Spec.Common.AutoCompactionRetention + } + commandArgs = append(commandArgs, fmt.Sprintf("--auto-compaction-retention=%s", compactionRetention)) + + // Client and Backup TLS command line args + // ----------------------------------------------------------------------------------------------------------------- + if b.etcd.Spec.Etcd.ClientUrlTLS != nil { + commandArgs = append(commandArgs, "--cert=/var/etcd/ssl/client/client/tls.crt") + commandArgs = append(commandArgs, "--key=/var/etcd/ssl/client/client/tls.key") + dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") + commandArgs = append(commandArgs, fmt.Sprintf("--cacert=/var/etcd/ssl/client/ca/%s", dataKey)) + commandArgs = append(commandArgs, "--insecure-transport=false") + commandArgs = append(commandArgs, "--insecure-skip-tls-verify=false") + commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, defaultClientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=https://%s:%d", b.etcd.GetClientServiceName(), defaultClientPort)) + } else { + commandArgs = append(commandArgs, "--insecure-transport=true") + commandArgs = append(commandArgs, "--insecure-skip-tls-verify=true") + commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=http://%s-local:%d", b.etcd.Name, defaultClientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), defaultClientPort)) + } + if b.etcd.Spec.Backup.TLS != nil { + commandArgs = append(commandArgs, "--server-cert=/var/etcd/ssl/client/server/tls.crt") + commandArgs = append(commandArgs, "--server-key=/var/etcd/ssl/client/server/tls.key") + } + + // Other misc command line args + // ----------------------------------------------------------------------------------------------------------------- + commandArgs = append(commandArgs, "--data-dir=/var/etcd/data/new.etcd") + commandArgs = append(commandArgs, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-connection-timeout=%s", defaultEtcdConnectionTimeout)) + + var quota = defaultQuota + if b.etcd.Spec.Etcd.Quota != nil { + quota = b.etcd.Spec.Etcd.Quota.Value() + } + commandArgs = append(commandArgs, fmt.Sprintf("--embedded-etcd-quota-bytes=%d", quota)) + if utils.TypeDeref[bool](b.etcd.Spec.Backup.EnableProfiling, false) { + commandArgs = append(commandArgs, "--enable-profiling=true") + } + + heartbeatDuration := defaultHeartbeatDuration + if b.etcd.Spec.Etcd.HeartbeatDuration != nil { + heartbeatDuration = b.etcd.Spec.Etcd.HeartbeatDuration.Duration.String() + } + commandArgs = append(commandArgs, fmt.Sprintf("--k8s-heartbeat-duration=%s", heartbeatDuration)) + + if b.etcd.Spec.Backup.LeaderElection != nil { + if b.etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-connection-timeout-leader-election=%s", b.etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout.Duration.String())) + } + if b.etcd.Spec.Backup.LeaderElection.ReelectionPeriod != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--reelection-period=%s", b.etcd.Spec.Backup.LeaderElection.ReelectionPeriod.Duration.String())) + } + } + + return commandArgs +} + +func (b *stsBuilder) getBackupStoreCommandArgs() []string { + var commandArgs []string + + commandArgs = append(commandArgs, "--enable-snapshot-lease-renewal=true") + commandArgs = append(commandArgs, fmt.Sprintf("--storage-provider=%s", *b.provider)) + commandArgs = append(commandArgs, fmt.Sprintf("--store-prefix=%s", b.etcd.Spec.Backup.Store.Prefix)) + + // Full snapshot command line args + // ----------------------------------------------------------------------------------------------------------------- + commandArgs = append(commandArgs, fmt.Sprintf("--full-snapshot-lease-name=%s", b.etcd.GetFullSnapshotLeaseName())) + if b.etcd.Spec.Backup.FullSnapshotSchedule != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--schedule=%s", *b.etcd.Spec.Backup.FullSnapshotSchedule)) + } + + // Delta snapshot command line args + // ----------------------------------------------------------------------------------------------------------------- + commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-lease-name=%s", b.etcd.GetDeltaSnapshotLeaseName())) + if b.etcd.Spec.Backup.DeltaSnapshotPeriod != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-period=%s", b.etcd.Spec.Backup.DeltaSnapshotPeriod.Duration.String())) + } + if b.etcd.Spec.Backup.DeltaSnapshotRetentionPeriod != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-retention-period=%s", b.etcd.Spec.Backup.DeltaSnapshotRetentionPeriod.Duration.String())) + } + var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit + if b.etcd.Spec.Backup.DeltaSnapshotMemoryLimit != nil { + deltaSnapshotMemoryLimit = b.etcd.Spec.Backup.DeltaSnapshotMemoryLimit.Value() + } + commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-memory-limit=%d", deltaSnapshotMemoryLimit)) + + // garbage collection command line args + // ----------------------------------------------------------------------------------------------------------------- + garbageCollectionPolicy := defaultGbcPolicy + if b.etcd.Spec.Backup.GarbageCollectionPolicy != nil { + garbageCollectionPolicy = string(*b.etcd.Spec.Backup.GarbageCollectionPolicy) + } + commandArgs = append(commandArgs, fmt.Sprintf("--garbage-collection-policy=%s", garbageCollectionPolicy)) + if garbageCollectionPolicy == "LimitBased" { + commandArgs = append(commandArgs, "--max-backups=7") + } + if b.etcd.Spec.Backup.GarbageCollectionPeriod != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--garbage-collection-period=%s", b.etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String())) + } + + // Snapshot compression and timeout command line args + // ----------------------------------------------------------------------------------------------------------------- + if b.etcd.Spec.Backup.SnapshotCompression != nil { + if utils.TypeDeref[bool](b.etcd.Spec.Backup.SnapshotCompression.Enabled, false) { + commandArgs = append(commandArgs, fmt.Sprintf("--compress-snapshots=%t", *b.etcd.Spec.Backup.SnapshotCompression.Enabled)) + } + if b.etcd.Spec.Backup.SnapshotCompression.Policy != nil { + commandArgs = append(commandArgs, fmt.Sprintf("--compression-policy=%s", string(*b.etcd.Spec.Backup.SnapshotCompression.Policy))) + } + } + + etcdSnapshotTimeout := defaultEtcdSnapshotTimeout + if b.etcd.Spec.Backup.EtcdSnapshotTimeout != nil { + etcdSnapshotTimeout = b.etcd.Spec.Backup.EtcdSnapshotTimeout.Duration.String() + } + commandArgs = append(commandArgs, "--etcd-snapshot-timeout="+etcdSnapshotTimeout) + + return commandArgs +} + +func (b *stsBuilder) getEtcdContainerReadinessProbe() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: b.getEtcdContainerReadinessHandler(), + InitialDelaySeconds: 15, + PeriodSeconds: 5, + FailureThreshold: 5, + } +} + +func (b *stsBuilder) getEtcdContainerReadinessHandler() corev1.ProbeHandler { + multiNodeCluster := b.etcd.Spec.Replicas > 1 + if multiNodeCluster && !b.useEtcdWrapper { + return corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: b.getEtcdContainerReadinessProbeCommand(), + }, + } + } + scheme := utils.IfConditionOr[corev1.URIScheme](b.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) + path := utils.IfConditionOr[string](multiNodeCluster, "/readyz", "/healthz") + port := utils.IfConditionOr[int](multiNodeCluster, defaultWrapperPort, int(defaultBackupPort)) + + return corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: path, + Port: intstr.FromInt(port), + Scheme: scheme, + }, + } +} + +func (b *stsBuilder) getEtcdContainerReadinessProbeCommand() []string { + cmdBuilder := strings.Builder{} + cmdBuilder.WriteString("ETCDCTL_API=3 etcdctl") + if b.etcd.Spec.Etcd.ClientUrlTLS != nil { + dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") + cmdBuilder.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) + cmdBuilder.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") + cmdBuilder.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") + cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", b.etcd.Name, defaultClientPort)) + } else { + cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", b.etcd.Name, defaultClientPort)) + } + cmdBuilder.WriteString(" get foo") + cmdBuilder.WriteString(" --consistency=l") + + return []string{ + "/bin/sh", + "-ec", + cmdBuilder.String(), + } +} + +func (b *stsBuilder) getEtcdContainerCommandArgs() []string { + if !b.useEtcdWrapper { + // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh + return []string{} + } + commandArgs := []string{"start-etcd"} + commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", b.etcd.Name)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-server-name=%s-local", b.etcd.Name)) + + if b.etcd.Spec.Etcd.ClientUrlTLS == nil { + commandArgs = append(commandArgs, "--backup-restore-tls-enabled=false") + } else { + dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") + commandArgs = append(commandArgs, "--backup-restore-tls-enabled=true") + commandArgs = append(commandArgs, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") + commandArgs = append(commandArgs, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") + commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) + } + return commandArgs +} + +func (b *stsBuilder) getEtcdContainerEnvVars() []corev1.EnvVar { + if b.useEtcdWrapper { + return []corev1.EnvVar{} + } + backTLSEnabled := b.etcd.Spec.Backup.TLS != nil + scheme := utils.IfConditionOr[string](backTLSEnabled, "https", "http") + endpoint := fmt.Sprintf("%s://%s-local:%d", scheme, b.etcd.Name, defaultBackupPort) + + return []corev1.EnvVar{ + {Name: "ENABLE_TLS", Value: strconv.FormatBool(backTLSEnabled)}, + {Name: "BACKUP_ENDPOINT", Value: endpoint}, + } +} + +func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { + if !b.useEtcdWrapper { + return nil + } + return &corev1.PodSecurityContext{ + RunAsGroup: pointer.Int64(65532), + RunAsNonRoot: pointer.Bool(true), + RunAsUser: pointer.Int64(65532), + } +} + +func (b *stsBuilder) getSecretVolumeMounts() []corev1.VolumeMount { + secretVolumeMounts := make([]corev1.VolumeMount, 0, 5) + if b.etcd.Spec.Etcd.ClientUrlTLS != nil { + secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ + Name: "client-url-ca-etcd", + MountPath: "/var/etcd/ssl/client/ca", + }, corev1.VolumeMount{ + Name: "client-url-etcd-server-tls", + MountPath: "/var/etcd/ssl/client/server", + }, corev1.VolumeMount{ + Name: "client-url-etcd-client-tls", + MountPath: "/var/etcd/ssl/client/client", + }) + } + if b.etcd.Spec.Etcd.PeerUrlTLS != nil { + secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ + Name: "peer-url-ca-etcd", + MountPath: "/var/etcd/ssl/peer/ca", + }, corev1.VolumeMount{ + Name: "peer-url-etcd-server-tls", + MountPath: "/var/etcd/ssl/peer/server", + }) + } + return secretVolumeMounts +} + +// getPodVolumes gets volumes that needs to be mounted onto the etcd StatefulSet pods +func (b *stsBuilder) getPodVolumes(ctx resource.OperatorContext) ([]corev1.Volume, error) { + volumes := []corev1.Volume{ + { + Name: "etcd-config-file", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: b.etcd.GetConfigMapName(), + }, + Items: []corev1.KeyToPath{ + { + Key: "etcd.conf.yaml", + Path: "etcd.conf.yaml", + }, + }, + DefaultMode: pointer.Int32(0644), + }, + }, + }, + } + + if b.etcd.Spec.Etcd.ClientUrlTLS != nil { + volumes = append(volumes, b.getClientTLSVolumes()...) + } + if b.etcd.Spec.Etcd.PeerUrlTLS != nil { + volumes = append(volumes, b.getPeerTLSVolumes()...) + } + if b.etcd.IsBackupStoreEnabled() { + backupVolume, err := b.getBackupVolume(ctx) + if err != nil { + return nil, err + } + if backupVolume != nil { + volumes = append(volumes, *backupVolume) + } + } + return volumes, nil +} + +func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { + clientTLSConfig := b.etcd.Spec.Etcd.ClientUrlTLS + return []corev1.Volume{ + { + Name: "client-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: clientTLSConfig.TLSCASecretRef.Name, + }, + }, + }, + { + Name: "client-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: clientTLSConfig.ServerTLSSecretRef.Name, + }, + }, + }, + { + Name: "client-url-etcd-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: clientTLSConfig.ClientTLSSecretRef.Name, + }, + }, + }, + } +} + +func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { + peerTLSConfig := b.etcd.Spec.Etcd.PeerUrlTLS + return []corev1.Volume{ + { + Name: "peer-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: peerTLSConfig.TLSCASecretRef.Name, + }, + }, + }, + { + Name: "peer-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: peerTLSConfig.ServerTLSSecretRef.Name, + }, + }, + }, + } +} + +func (b *stsBuilder) getBackupVolume(ctx resource.OperatorContext) (*corev1.Volume, error) { + if b.provider == nil { + return nil, nil + } + store := b.etcd.Spec.Backup.Store + switch *b.provider { + case "Local": + hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, b.client, b.logger, store, b.etcd.GetNamespace()) + if err != nil { + return nil, fmt.Errorf("error getting host mount path for etcd: %v Err: %w", b.etcd.GetNamespaceName(), err) + } + + hpt := corev1.HostPathDirectory + return &corev1.Volume{ + Name: "host-storage", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath + "/" + pointer.StringDeref(store.Container, ""), + Type: &hpt, + }, + }, + }, nil + case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + if store.SecretRef == nil { + return nil, fmt.Errorf("etcd: %v, no secretRef configured for backup store", b.etcd.GetNamespaceName()) + } + + return &corev1.Volume{ + Name: "etcd-backup", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: store.SecretRef.Name, + }, + }, + }, nil + } + return nil, nil +} + +func getBackupStoreProvider(etcd *druidv1alpha1.Etcd) (*string, error) { + if !etcd.IsBackupStoreEnabled() { + return nil, nil + } + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return nil, err + } + return &provider, nil +} diff --git a/internal/operator/statefulset/helper.go b/internal/operator/statefulset/helper.go index 57753917b..6b6772884 100644 --- a/internal/operator/statefulset/helper.go +++ b/internal/operator/statefulset/helper.go @@ -1,870 +1,870 @@ package statefulset -import ( - "encoding/json" - "fmt" - "strconv" - "strings" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/operator/resource" - "github.com/gardener/etcd-druid/internal/utils" - gardenerutils "github.com/gardener/gardener/pkg/utils" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apiresource "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type consistencyLevel string - -const ( - linearizable consistencyLevel = "linearizable" - serializable consistencyLevel = "serializable" - defaultBackupPort int32 = 8080 - defaultServerPort int32 = 2380 - defaultClientPort int32 = 2379 - defaultWrapperPort int32 = 9095 - defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi - defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi - defaultHeartbeatDuration = "10s" - defaultGbcPolicy = "LimitBased" - defaultAutoCompactionRetention = "30m" - defaultEtcdSnapshotTimeout = "15m" - defaultEtcdDefragTimeout = "15m" - defaultAutoCompactionMode = "periodic" - defaultEtcdConnectionTimeout = "5m" -) - -var defaultStorageCapacity = apiresource.MustParse("16Gi") -var defaultResourceRequirements = corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: apiresource.MustParse("50m"), - corev1.ResourceMemory: apiresource.MustParse("128Mi"), - }, -} - -func extractObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Name: etcd.Name, - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - } -} -func extractPodObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd, configMapChecksum string) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Labels: utils.MergeStringMaps(make(map[string]string), etcd.Spec.Labels, etcd.GetDefaultLabels()), - Annotations: utils.MergeStringMaps(map[string]string{ - resource.ConfigMapCheckSumKey: configMapChecksum, - }, etcd.Spec.Annotations), - } -} - -func getEtcdCommandArgs(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []string { - if !useEtcdWrapper { - // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh - return []string{} - } - //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA - command := []string{"" + "start-etcd"} - command = append(command, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", etcd.Name)) - command = append(command, fmt.Sprintf("--etcd-server-name=%s-local", etcd.Name)) - - clientURLTLS := etcd.Spec.Etcd.ClientUrlTLS - if clientURLTLS == nil { - command = append(command, "--backup-restore-tls-enabled=false") - } else { - dataKey := "ca.crt" - if clientURLTLS.TLSCASecretRef.DataKey != nil { - dataKey = *clientURLTLS.TLSCASecretRef.DataKey - } - command = append(command, "--backup-restore-tls-enabled=true") - command = append(command, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") - command = append(command, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") - command = append(command, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) - } - - return command -} - -func getReadinessHandler(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { - if etcd.Spec.Replicas > 1 { - // TODO(timuthy): Special handling for multi-node etcd can be removed as soon as - // etcd-backup-restore supports `/healthz` for etcd followers, see https://github.com/gardener/etcd-backup-restore/pull/491. - return getReadinessHandlerForMultiNode(useEtcdWrapper, etcd) - } - return getReadinessHandlerForSingleNode(etcd) -} - -func getReadinessHandlerForSingleNode(etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { - scheme := corev1.URISchemeHTTPS - if etcd.Spec.Backup.TLS == nil { - scheme = corev1.URISchemeHTTP - } - - return corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(int(defaultBackupPort)), - Scheme: scheme, - }, - } -} - -func getReadinessHandlerForMultiNode(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { - if useEtcdWrapper { - //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA - scheme := corev1.URISchemeHTTPS - if etcd.Spec.Backup.TLS == nil { - scheme = corev1.URISchemeHTTP - } - - return corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromInt(int(defaultWrapperPort)), - Scheme: scheme, - }, - } - } - - return corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: getProbeCommand(etcd, linearizable), - }, - } -} - -func getEtcdPorts() []corev1.ContainerPort { - return []corev1.ContainerPort{ - { - Name: "server", - Protocol: "TCP", - ContainerPort: defaultServerPort, - }, - { - Name: "client", - Protocol: "TCP", - ContainerPort: defaultClientPort, - }, - } -} - -func getProbeCommand(etcd *druidv1alpha1.Etcd, consistency consistencyLevel) []string { - var etcdCtlCommand strings.Builder - - etcdCtlCommand.WriteString("ETCDCTL_API=3 etcdctl") - - if etcd.Spec.Etcd.ClientUrlTLS != nil { - dataKey := "ca.crt" - if etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey != nil { - dataKey = *etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey - } - - etcdCtlCommand.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) - etcdCtlCommand.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") - etcdCtlCommand.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") - etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) - - } else { - etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) - } - - etcdCtlCommand.WriteString(" get foo") - - switch consistency { - case linearizable: - etcdCtlCommand.WriteString(" --consistency=l") - case serializable: - etcdCtlCommand.WriteString(" --consistency=s") - } - - return []string{ - "/bin/sh", - "-ec", - etcdCtlCommand.String(), - } -} - -func getEtcdResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { - if etcd.Spec.Etcd.Resources != nil { - return *etcd.Spec.Etcd.Resources - } - - return defaultResourceRequirements -} - -func getEtcdEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { - protocol := "http" - if etcd.Spec.Backup.TLS != nil { - protocol = "https" - } - - endpoint := fmt.Sprintf("%s://%s-local:%d", protocol, etcd.Name, defaultBackupPort) - - return []corev1.EnvVar{ - getEnvVarFromValue("ENABLE_TLS", strconv.FormatBool(etcd.Spec.Backup.TLS != nil)), - getEnvVarFromValue("BACKUP_ENDPOINT", endpoint), - } -} - -func getEnvVarFromValue(name, value string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - Value: value, - } -} - -func getEnvVarFromField(name, fieldPath string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: fieldPath, - }, - }, - } -} - -func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, - }, - Key: secretKey, - }, - }, - } -} - -func getEtcdVolumeMounts(etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { - volumeClaimTemplateName := etcd.Name - if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { - volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate - } - - vms := []corev1.VolumeMount{ - { - Name: volumeClaimTemplateName, - MountPath: "/var/etcd/data/", - }, - } - - vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) - - return vms -} - -func getSecretVolumeMounts(clientUrlTLS, peerUrlTLS *druidv1alpha1.TLSConfig) []corev1.VolumeMount { - var vms []corev1.VolumeMount - - if clientUrlTLS != nil { - vms = append(vms, corev1.VolumeMount{ - Name: "client-url-ca-etcd", - MountPath: "/var/etcd/ssl/client/ca", - }, corev1.VolumeMount{ - Name: "client-url-etcd-server-tls", - MountPath: "/var/etcd/ssl/client/server", - }, corev1.VolumeMount{ - Name: "client-url-etcd-client-tls", - MountPath: "/var/etcd/ssl/client/client", - }) - } - - if peerUrlTLS != nil { - vms = append(vms, corev1.VolumeMount{ - Name: "peer-url-ca-etcd", - MountPath: "/var/etcd/ssl/peer/ca", - }, corev1.VolumeMount{ - Name: "peer-url-etcd-server-tls", - MountPath: "/var/etcd/ssl/peer/server", - }) - } - - return vms -} - -func getBackupRestoreCommandArgs(etcd *druidv1alpha1.Etcd) []string { - command := []string{"server"} - - if etcd.Spec.Backup.Store != nil { - command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) - command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) - } - - if etcd.Spec.Etcd.DefragmentationSchedule != nil { - command = append(command, "--defragmentation-schedule="+*etcd.Spec.Etcd.DefragmentationSchedule) - } - - if etcd.Spec.Backup.FullSnapshotSchedule != nil { - command = append(command, "--schedule="+*etcd.Spec.Backup.FullSnapshotSchedule) - } - - garbageCollectionPolicy := defaultGbcPolicy - if etcd.Spec.Backup.GarbageCollectionPolicy != nil { - garbageCollectionPolicy = string(*etcd.Spec.Backup.GarbageCollectionPolicy) - } - - command = append(command, "--garbage-collection-policy="+garbageCollectionPolicy) - if garbageCollectionPolicy == "LimitBased" { - command = append(command, "--max-backups=7") - } - - command = append(command, "--data-dir=/var/etcd/data/new.etcd") - command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") - - if etcd.Spec.Backup.Store != nil { - store, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err != nil { - return nil - } - command = append(command, "--storage-provider="+store) - command = append(command, "--store-prefix="+string(etcd.Spec.Backup.Store.Prefix)) - } - - var quota = defaultQuota - if etcd.Spec.Etcd.Quota != nil { - quota = etcd.Spec.Etcd.Quota.Value() - } - - command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) - - if pointer.BoolDeref(etcd.Spec.Backup.EnableProfiling, false) { - command = append(command, "--enable-profiling=true") - } - - if etcd.Spec.Etcd.ClientUrlTLS != nil { - command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") - command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") - command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+pointer.StringDeref(etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt")) - command = append(command, "--insecure-transport=false") - command = append(command, "--insecure-skip-tls-verify=false") - command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) - command = append(command, fmt.Sprintf("--service-endpoints=https://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) - } else { - command = append(command, "--insecure-transport=true") - command = append(command, "--insecure-skip-tls-verify=true") - command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) - command = append(command, fmt.Sprintf("--service-endpoints=http://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) - - } - - if etcd.Spec.Backup.TLS != nil { - command = append(command, "--server-cert=/var/etcd/ssl/client/server/tls.crt") - command = append(command, "--server-key=/var/etcd/ssl/client/server/tls.key") - } - - command = append(command, "--etcd-connection-timeout="+defaultEtcdConnectionTimeout) - - if etcd.Spec.Backup.DeltaSnapshotPeriod != nil { - command = append(command, "--delta-snapshot-period="+etcd.Spec.Backup.DeltaSnapshotPeriod.Duration.String()) - } - - if etcd.Spec.Backup.DeltaSnapshotRetentionPeriod != nil { - command = append(command, "--delta-snapshot-retention-period="+etcd.Spec.Backup.DeltaSnapshotRetentionPeriod.Duration.String()) - } - - var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit - if etcd.Spec.Backup.DeltaSnapshotMemoryLimit != nil { - deltaSnapshotMemoryLimit = etcd.Spec.Backup.DeltaSnapshotMemoryLimit.Value() - } - - command = append(command, "--delta-snapshot-memory-limit="+fmt.Sprint(deltaSnapshotMemoryLimit)) - - if etcd.Spec.Backup.GarbageCollectionPeriod != nil { - command = append(command, "--garbage-collection-period="+etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String()) - } - - if etcd.Spec.Backup.SnapshotCompression != nil { - if pointer.BoolDeref(etcd.Spec.Backup.SnapshotCompression.Enabled, false) { - command = append(command, "--compress-snapshots="+fmt.Sprint(*etcd.Spec.Backup.SnapshotCompression.Enabled)) - } - if etcd.Spec.Backup.SnapshotCompression.Policy != nil { - command = append(command, "--compression-policy="+string(*etcd.Spec.Backup.SnapshotCompression.Policy)) - } - } - - compactionMode := defaultAutoCompactionMode - if etcd.Spec.Common.AutoCompactionMode != nil { - compactionMode = string(*etcd.Spec.Common.AutoCompactionMode) - } - command = append(command, "--auto-compaction-mode="+compactionMode) - - compactionRetention := defaultAutoCompactionRetention - if etcd.Spec.Common.AutoCompactionRetention != nil { - compactionRetention = *etcd.Spec.Common.AutoCompactionRetention - } - command = append(command, "--auto-compaction-retention="+compactionRetention) - - etcdSnapshotTimeout := defaultEtcdSnapshotTimeout - if etcd.Spec.Backup.EtcdSnapshotTimeout != nil { - etcdSnapshotTimeout = etcd.Spec.Backup.EtcdSnapshotTimeout.Duration.String() - } - command = append(command, "--etcd-snapshot-timeout="+etcdSnapshotTimeout) - - etcdDefragTimeout := defaultEtcdDefragTimeout - if etcd.Spec.Etcd.EtcdDefragTimeout != nil { - etcdDefragTimeout = etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String() - } - command = append(command, "--etcd-defrag-timeout="+etcdDefragTimeout) - - command = append(command, "--snapstore-temp-directory=/var/etcd/data/temp") - command = append(command, "--enable-member-lease-renewal=true") - - heartbeatDuration := defaultHeartbeatDuration - if etcd.Spec.Etcd.HeartbeatDuration != nil { - heartbeatDuration = etcd.Spec.Etcd.HeartbeatDuration.Duration.String() - } - command = append(command, "--k8s-heartbeat-duration="+heartbeatDuration) - - if etcd.Spec.Backup.LeaderElection != nil { - if etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout != nil { - command = append(command, "--etcd-connection-timeout-leader-election="+etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout.Duration.String()) - } - - if etcd.Spec.Backup.LeaderElection.ReelectionPeriod != nil { - command = append(command, "--reelection-period="+etcd.Spec.Backup.LeaderElection.ReelectionPeriod.Duration.String()) - } - } - - return command -} - -func getBackupPorts() []corev1.ContainerPort { - return []corev1.ContainerPort{ - { - Name: "server", - Protocol: "TCP", - ContainerPort: defaultBackupPort, - }, - } -} - -func getBackupResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { - if etcd.Spec.Backup.Resources != nil { - return *etcd.Spec.Backup.Resources - } - return defaultResourceRequirements -} - -func getBackupRestoreEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { - var ( - env []corev1.EnvVar - storageContainer string - storeValues = etcd.Spec.Backup.Store - ) - - if etcd.Spec.Backup.Store != nil { - storageContainer = pointer.StringDeref(etcd.Spec.Backup.Store.Container, "") - } - - // TODO(timuthy, shreyas-s-rao): Move STORAGE_CONTAINER a few lines below so that we can append and exit in one step. This should only be done in a release where a restart of etcd is acceptable. - env = append(env, getEnvVarFromValue("STORAGE_CONTAINER", storageContainer)) - env = append(env, getEnvVarFromField("POD_NAME", "metadata.name")) - env = append(env, getEnvVarFromField("POD_NAMESPACE", "metadata.namespace")) - - if storeValues == nil { - return env - } - - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err != nil { - return env - } - - // TODO(timuthy): move this to a non root path when we switch to a rootless distribution - const credentialsMountPath = "/var/etcd-backup" - switch provider { - case utils.S3: - env = append(env, getEnvVarFromValue("AWS_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.ABS: - env = append(env, getEnvVarFromValue("AZURE_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.GCS: - env = append(env, getEnvVarFromValue("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) - - case utils.Swift: - env = append(env, getEnvVarFromValue("OPENSTACK_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.OSS: - env = append(env, getEnvVarFromValue("ALICLOUD_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.ECS: - env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) - env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) - env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) - - case utils.OCS: - env = append(env, getEnvVarFromValue("OPENSHIFT_APPLICATION_CREDENTIALS", credentialsMountPath)) - } - - return env -} - -func getBackupRestoreVolumeMounts(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { - volumeClaimTemplateName := etcd.Name - if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { - volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate - } - vms := []corev1.VolumeMount{ - { - Name: volumeClaimTemplateName, - MountPath: "/var/etcd/data", - }, - { - Name: "etcd-config-file", - MountPath: "/var/etcd/config/", - }, - } - - vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) - - if etcd.Spec.Backup.Store == nil { - return vms - } - - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err != nil { - return vms - } - - switch provider { - case utils.Local: - if etcd.Spec.Backup.Store.Container != nil { - if useEtcdWrapper { - vms = append(vms, corev1.VolumeMount{ - Name: "host-storage", - MountPath: "/home/nonroot/" + pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), - }) - } else { - vms = append(vms, corev1.VolumeMount{ - Name: "host-storage", - MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), - }) - } - } - case utils.GCS: - vms = append(vms, corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/.gcp/", - }) - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - vms = append(vms, corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/etcd-backup/", - }) - } - - return vms -} - -func getvolumeClaimTemplateName(etcd *druidv1alpha1.Etcd) string { - volumeClaimTemplateName := etcd.Name - if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { - volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate - } - return volumeClaimTemplateName -} - -func getStorageReq(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { - storageCapacity := defaultStorageCapacity - if etcd.Spec.StorageCapacity != nil { - storageCapacity = *etcd.Spec.StorageCapacity - } - - return corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: storageCapacity, - }, - } -} - -func addEtcdContainer(etcdImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { - return &corev1.Container{ - Name: "etcd", - Image: *etcdImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: getEtcdCommandArgs(isEtcdWrapperEnabled, etcd), - ReadinessProbe: &corev1.Probe{ - ProbeHandler: getReadinessHandler(isEtcdWrapperEnabled, etcd), - InitialDelaySeconds: 15, - PeriodSeconds: 5, - FailureThreshold: 5, - }, - Ports: getEtcdPorts(), - Resources: getEtcdResources(etcd), - Env: getEtcdEnvVars(etcd), - VolumeMounts: getEtcdVolumeMounts(etcd), - } -} - -func addBackupRestoreContainer(etcdBackupImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { - return &corev1.Container{ - Name: "backup-restore", - Image: *etcdBackupImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: getBackupRestoreCommandArgs(etcd), - Ports: getBackupPorts(), - Resources: getBackupResources(etcd), - Env: getBackupRestoreEnvVars(etcd), - VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_PTRACE", - }, - }, - }, - } -} - -func addInitContainersIfWrapperEnabled(initContainerImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) []corev1.Container { - if !isEtcdWrapperEnabled { - return []corev1.Container{} - } - - // Initialize the slice with the 'change-permissions' container - initContainers := []corev1.Container{ - { - Name: "change-permissions", - Image: *initContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{"chown -R 65532:65532 /var/etcd/data"}, - VolumeMounts: getEtcdVolumeMounts(etcd), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }, - } - - if etcd.Spec.Backup.Store != nil { - prov, _ := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if prov == utils.Local { - initContainers = append(initContainers, corev1.Container{ - Name: "change-backup-bucket-permissions", - Image: *initContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, - VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }) - } - } - - return initContainers -} - -func getVolumeClaimTemplates(etcd *druidv1alpha1.Etcd) []corev1.PersistentVolumeClaim { - return []corev1.PersistentVolumeClaim{{ - ObjectMeta: metav1.ObjectMeta{ - Name: getvolumeClaimTemplateName(etcd), - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: getStorageReq(etcd), - StorageClassName: etcd.Spec.StorageClass, - }, - }, - } -} - -func addSecurityContextIfWrapperEnabled(isEtcdWrapperEnabled bool) *corev1.PodSecurityContext { - return &corev1.PodSecurityContext{ - RunAsGroup: pointer.Int64(65532), - RunAsNonRoot: pointer.Bool(true), - RunAsUser: pointer.Int64(65532), - } -} - -func getPriorityClassName(etcd *druidv1alpha1.Etcd) string { - if etcd.Spec.PriorityClassName != nil { - return *etcd.Spec.PriorityClassName - } - return "" -} - -// hasImmutableFieldChanged checks if any immutable fields have changed in the StatefulSet -// specification compared to the Etcd object. It returns true if there are changes in the immutable fields. -// Currently, it checks for changes in the ServiceName and PodManagementPolicy. -func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { - return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement -} - -func getVolumes(client client.Client, logger logr.Logger, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { - vs := []corev1.Volume{ - { - Name: "etcd-config-file", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: etcd.GetConfigMapName(), - }, - Items: []corev1.KeyToPath{ - { - Key: "etcd.conf.yaml", - Path: "etcd.conf.yaml", - }, - }, - DefaultMode: pointer.Int32(0644), - }, - }, - }, - } - - if etcd.Spec.Etcd.ClientUrlTLS != nil { - vs = append(vs, corev1.Volume{ - Name: "client-url-ca-etcd", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "client-url-etcd-server-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "client-url-etcd-client-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - }, - }, - }) - } - - if etcd.Spec.Etcd.PeerUrlTLS != nil { - vs = append(vs, corev1.Volume{ - Name: "peer-url-ca-etcd", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "peer-url-etcd-server-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, - }, - }, - }) - } - - if etcd.Spec.Backup.Store == nil { - return vs, nil - } - - storeValues := etcd.Spec.Backup.Store - provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) - if err != nil { - return vs, nil - } - - switch provider { - case "Local": - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, client, logger, storeValues, etcd.GetNamespace()) - if err != nil { - return nil, err - } - - hpt := corev1.HostPathDirectory - vs = append(vs, corev1.Volume{ - Name: "host-storage", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), - Type: &hpt, - }, - }, - }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: - if storeValues.SecretRef == nil { - return nil, fmt.Errorf("no secretRef configured for backup store") - } - - vs = append(vs, corev1.Volume{ - Name: "etcd-backup", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, - }, - }, - }) - } - - return vs, nil -} - -func getConfigMapChecksum(cl client.Client, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (string, error) { - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetConfigMapName(), - Namespace: etcd.Namespace, - }, - } - if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { - - return "", err - } - jsonString, err := json.Marshal(cm.Data) - if err != nil { - return "", err - } - - return gardenerutils.ComputeSHA256Hex(jsonString), nil -} - -func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger logr.Logger, opName string, sts *appsv1.StatefulSet) error { - // Get all Pods belonging to the StatefulSet - podList := &corev1.PodList{} - listOpts := []client.ListOption{ - client.InNamespace(sts.Namespace), - client.MatchingLabels(sts.Spec.Selector.MatchLabels), - } - - if err := cl.List(ctx, podList, listOpts...); err != nil { - logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) - return err - } - - for _, pod := range podList.Items { - if err := cl.Delete(ctx, &pod); err != nil { - logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) - return err - } - } - - return nil -} - -// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled -func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { - return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil -} +//import ( +// "encoding/json" +// "fmt" +// "strconv" +// "strings" +// +// druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +// "github.com/gardener/etcd-druid/internal/operator/resource" +// "github.com/gardener/etcd-druid/internal/utils" +// gardenerutils "github.com/gardener/gardener/pkg/utils" +// "github.com/go-logr/logr" +// appsv1 "k8s.io/api/apps/v1" +// corev1 "k8s.io/api/core/v1" +// apiresource "k8s.io/apimachinery/pkg/api/resource" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// "k8s.io/apimachinery/pkg/util/intstr" +// "k8s.io/utils/pointer" +// "sigs.k8s.io/controller-runtime/pkg/client" +//) +// +//type consistencyLevel string +// +//const ( +// linearizable consistencyLevel = "linearizable" +// serializable consistencyLevel = "serializable" +// defaultBackupPort int32 = 8080 +// defaultServerPort int32 = 2380 +// defaultClientPort int32 = 2379 +// defaultWrapperPort int32 = 9095 +// defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi +// defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi +// defaultHeartbeatDuration = "10s" +// defaultGbcPolicy = "LimitBased" +// defaultAutoCompactionRetention = "30m" +// defaultEtcdSnapshotTimeout = "15m" +// defaultEtcdDefragTimeout = "15m" +// defaultAutoCompactionMode = "periodic" +// defaultEtcdConnectionTimeout = "5m" +//) +// +//var defaultStorageCapacity = apiresource.MustParse("16Gi") +//var defaultResourceRequirements = corev1.ResourceRequirements{ +// Requests: corev1.ResourceList{ +// corev1.ResourceCPU: apiresource.MustParse("50m"), +// corev1.ResourceMemory: apiresource.MustParse("128Mi"), +// }, +//} +// +//func extractObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { +// return metav1.ObjectMeta{ +// Name: etcd.Name, +// Namespace: etcd.Namespace, +// Labels: etcd.GetDefaultLabels(), +// OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, +// } +//} +//func extractPodObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd, configMapChecksum string) metav1.ObjectMeta { +// return metav1.ObjectMeta{ +// Labels: utils.MergeStringMaps(make(map[string]string), etcd.Spec.Labels, etcd.GetDefaultLabels()), +// Annotations: utils.MergeStringMaps(map[string]string{ +// resource.ConfigMapCheckSumKey: configMapChecksum, +// }, etcd.Spec.Annotations), +// } +//} +// +//func getEtcdCommandArgs(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []string { +// if !useEtcdWrapper { +// // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh +// return []string{} +// } +// //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA +// command := []string{"" + "start-etcd"} +// command = append(command, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", etcd.Name)) +// command = append(command, fmt.Sprintf("--etcd-server-name=%s-local", etcd.Name)) +// +// clientURLTLS := etcd.Spec.Etcd.ClientUrlTLS +// if clientURLTLS == nil { +// command = append(command, "--backup-restore-tls-enabled=false") +// } else { +// dataKey := "ca.crt" +// if clientURLTLS.TLSCASecretRef.DataKey != nil { +// dataKey = *clientURLTLS.TLSCASecretRef.DataKey +// } +// command = append(command, "--backup-restore-tls-enabled=true") +// command = append(command, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") +// command = append(command, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") +// command = append(command, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) +// } +// +// return command +//} +// +//func getReadinessHandler(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { +// if etcd.Spec.Replicas > 1 { +// // TODO(timuthy): Special handling for multi-node etcd can be removed as soon as +// // etcd-backup-restore supports `/healthz` for etcd followers, see https://github.com/gardener/etcd-backup-restore/pull/491. +// return getReadinessHandlerForMultiNode(useEtcdWrapper, etcd) +// } +// return getReadinessHandlerForSingleNode(etcd) +//} +// +//func getReadinessHandlerForSingleNode(etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { +// scheme := corev1.URISchemeHTTPS +// if etcd.Spec.Backup.TLS == nil { +// scheme = corev1.URISchemeHTTP +// } +// +// return corev1.ProbeHandler{ +// HTTPGet: &corev1.HTTPGetAction{ +// Path: "/healthz", +// Port: intstr.FromInt(int(defaultBackupPort)), +// Scheme: scheme, +// }, +// } +//} +// +//func getReadinessHandlerForMultiNode(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { +// if useEtcdWrapper { +// //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA +// scheme := corev1.URISchemeHTTPS +// if etcd.Spec.Backup.TLS == nil { +// scheme = corev1.URISchemeHTTP +// } +// +// return corev1.ProbeHandler{ +// HTTPGet: &corev1.HTTPGetAction{ +// Path: "/readyz", +// Port: intstr.FromInt(int(defaultWrapperPort)), +// Scheme: scheme, +// }, +// } +// } +// +// return corev1.ProbeHandler{ +// Exec: &corev1.ExecAction{ +// Command: getProbeCommand(etcd, linearizable), +// }, +// } +//} +// +//func getEtcdPorts() []corev1.ContainerPort { +// return []corev1.ContainerPort{ +// { +// Name: "server", +// Protocol: "TCP", +// ContainerPort: defaultServerPort, +// }, +// { +// Name: "client", +// Protocol: "TCP", +// ContainerPort: defaultClientPort, +// }, +// } +//} +// +//func getProbeCommand(etcd *druidv1alpha1.Etcd, consistency consistencyLevel) []string { +// var etcdCtlCommand strings.Builder +// +// etcdCtlCommand.WriteString("ETCDCTL_API=3 etcdctl") +// +// if etcd.Spec.Etcd.ClientUrlTLS != nil { +// dataKey := "ca.crt" +// if etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey != nil { +// dataKey = *etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey +// } +// +// etcdCtlCommand.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) +// etcdCtlCommand.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") +// etcdCtlCommand.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") +// etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) +// +// } else { +// etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) +// } +// +// etcdCtlCommand.WriteString(" get foo") +// +// switch consistency { +// case linearizable: +// etcdCtlCommand.WriteString(" --consistency=l") +// case serializable: +// etcdCtlCommand.WriteString(" --consistency=s") +// } +// +// return []string{ +// "/bin/sh", +// "-ec", +// etcdCtlCommand.String(), +// } +//} +// +//func getEtcdResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { +// if etcd.Spec.Etcd.Resources != nil { +// return *etcd.Spec.Etcd.Resources +// } +// +// return defaultResourceRequirements +//} +// +//func getEtcdEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { +// protocol := "http" +// if etcd.Spec.Backup.TLS != nil { +// protocol = "https" +// } +// +// endpoint := fmt.Sprintf("%s://%s-local:%d", protocol, etcd.Name, defaultBackupPort) +// +// return []corev1.EnvVar{ +// getEnvVarFromValue("ENABLE_TLS", strconv.FormatBool(etcd.Spec.Backup.TLS != nil)), +// getEnvVarFromValue("BACKUP_ENDPOINT", endpoint), +// } +//} +// +//func getEnvVarFromValue(name, value string) corev1.EnvVar { +// return corev1.EnvVar{ +// Name: name, +// Value: value, +// } +//} +// +//func getEnvVarFromField(name, fieldPath string) corev1.EnvVar { +// return corev1.EnvVar{ +// Name: name, +// ValueFrom: &corev1.EnvVarSource{ +// FieldRef: &corev1.ObjectFieldSelector{ +// FieldPath: fieldPath, +// }, +// }, +// } +//} +// +//func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { +// return corev1.EnvVar{ +// Name: name, +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: secretName, +// }, +// Key: secretKey, +// }, +// }, +// } +//} +// +//func getEtcdVolumeMounts(etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { +// volumeClaimTemplateName := etcd.Name +// if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { +// volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate +// } +// +// vms := []corev1.VolumeMount{ +// { +// Name: volumeClaimTemplateName, +// MountPath: "/var/etcd/data/", +// }, +// } +// +// vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) +// +// return vms +//} +// +//func getSecretVolumeMounts(clientUrlTLS, peerUrlTLS *druidv1alpha1.TLSConfig) []corev1.VolumeMount { +// var vms []corev1.VolumeMount +// +// if clientUrlTLS != nil { +// vms = append(vms, corev1.VolumeMount{ +// Name: "client-url-ca-etcd", +// MountPath: "/var/etcd/ssl/client/ca", +// }, corev1.VolumeMount{ +// Name: "client-url-etcd-server-tls", +// MountPath: "/var/etcd/ssl/client/server", +// }, corev1.VolumeMount{ +// Name: "client-url-etcd-client-tls", +// MountPath: "/var/etcd/ssl/client/client", +// }) +// } +// +// if peerUrlTLS != nil { +// vms = append(vms, corev1.VolumeMount{ +// Name: "peer-url-ca-etcd", +// MountPath: "/var/etcd/ssl/peer/ca", +// }, corev1.VolumeMount{ +// Name: "peer-url-etcd-server-tls", +// MountPath: "/var/etcd/ssl/peer/server", +// }) +// } +// +// return vms +//} +// +//func getBackupRestoreCommandArgs(etcd *druidv1alpha1.Etcd) []string { +// command := []string{"server"} +// +// if etcd.Spec.Backup.Store != nil { +// command = append(command, "--enable-snapshot-lease-renewal=true") +// command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) +// command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) +// } +// +// if etcd.Spec.Etcd.DefragmentationSchedule != nil { +// command = append(command, "--defragmentation-schedule="+*etcd.Spec.Etcd.DefragmentationSchedule) +// } +// +// if etcd.Spec.Backup.FullSnapshotSchedule != nil { +// command = append(command, "--schedule="+*etcd.Spec.Backup.FullSnapshotSchedule) +// } +// +// garbageCollectionPolicy := defaultGbcPolicy +// if etcd.Spec.Backup.GarbageCollectionPolicy != nil { +// garbageCollectionPolicy = string(*etcd.Spec.Backup.GarbageCollectionPolicy) +// } +// +// command = append(command, "--garbage-collection-policy="+garbageCollectionPolicy) +// if garbageCollectionPolicy == "LimitBased" { +// command = append(command, "--max-backups=7") +// } +// +// command = append(command, "--data-dir=/var/etcd/data/new.etcd") +// command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") +// +// if etcd.Spec.Backup.Store != nil { +// store, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) +// if err != nil { +// return nil +// } +// command = append(command, "--storage-provider="+store) +// command = append(command, "--store-prefix="+string(etcd.Spec.Backup.Store.Prefix)) +// } +// +// var quota = defaultQuota +// if etcd.Spec.Etcd.Quota != nil { +// quota = etcd.Spec.Etcd.Quota.Value() +// } +// +// command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) +// +// if pointer.BoolDeref(etcd.Spec.Backup.EnableProfiling, false) { +// command = append(command, "--enable-profiling=true") +// } +// +// if etcd.Spec.Etcd.ClientUrlTLS != nil { +// command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") +// command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") +// command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+pointer.StringDeref(etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt")) +// command = append(command, "--insecure-transport=false") +// command = append(command, "--insecure-skip-tls-verify=false") +// command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) +// command = append(command, fmt.Sprintf("--service-endpoints=https://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) +// } else { +// command = append(command, "--insecure-transport=true") +// command = append(command, "--insecure-skip-tls-verify=true") +// command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) +// command = append(command, fmt.Sprintf("--service-endpoints=http://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) +// +// } +// +// if etcd.Spec.Backup.TLS != nil { +// command = append(command, "--server-cert=/var/etcd/ssl/client/server/tls.crt") +// command = append(command, "--server-key=/var/etcd/ssl/client/server/tls.key") +// } +// +// command = append(command, "--etcd-connection-timeout="+defaultEtcdConnectionTimeout) +// +// if etcd.Spec.Backup.DeltaSnapshotPeriod != nil { +// command = append(command, "--delta-snapshot-period="+etcd.Spec.Backup.DeltaSnapshotPeriod.Duration.String()) +// } +// +// if etcd.Spec.Backup.DeltaSnapshotRetentionPeriod != nil { +// command = append(command, "--delta-snapshot-retention-period="+etcd.Spec.Backup.DeltaSnapshotRetentionPeriod.Duration.String()) +// } +// +// var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit +// if etcd.Spec.Backup.DeltaSnapshotMemoryLimit != nil { +// deltaSnapshotMemoryLimit = etcd.Spec.Backup.DeltaSnapshotMemoryLimit.Value() +// } +// +// command = append(command, "--delta-snapshot-memory-limit="+fmt.Sprint(deltaSnapshotMemoryLimit)) +// +// if etcd.Spec.Backup.GarbageCollectionPeriod != nil { +// command = append(command, "--garbage-collection-period="+etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String()) +// } +// +// if etcd.Spec.Backup.SnapshotCompression != nil { +// if pointer.BoolDeref(etcd.Spec.Backup.SnapshotCompression.Enabled, false) { +// command = append(command, "--compress-snapshots="+fmt.Sprint(*etcd.Spec.Backup.SnapshotCompression.Enabled)) +// } +// if etcd.Spec.Backup.SnapshotCompression.Policy != nil { +// command = append(command, "--compression-policy="+string(*etcd.Spec.Backup.SnapshotCompression.Policy)) +// } +// } +// +// compactionMode := defaultAutoCompactionMode +// if etcd.Spec.Common.AutoCompactionMode != nil { +// compactionMode = string(*etcd.Spec.Common.AutoCompactionMode) +// } +// command = append(command, "--auto-compaction-mode="+compactionMode) +// +// compactionRetention := defaultAutoCompactionRetention +// if etcd.Spec.Common.AutoCompactionRetention != nil { +// compactionRetention = *etcd.Spec.Common.AutoCompactionRetention +// } +// command = append(command, "--auto-compaction-retention="+compactionRetention) +// +// etcdSnapshotTimeout := defaultEtcdSnapshotTimeout +// if etcd.Spec.Backup.EtcdSnapshotTimeout != nil { +// etcdSnapshotTimeout = etcd.Spec.Backup.EtcdSnapshotTimeout.Duration.String() +// } +// command = append(command, "--etcd-snapshot-timeout="+etcdSnapshotTimeout) +// +// etcdDefragTimeout := defaultEtcdDefragTimeout +// if etcd.Spec.Etcd.EtcdDefragTimeout != nil { +// etcdDefragTimeout = etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String() +// } +// command = append(command, "--etcd-defrag-timeout="+etcdDefragTimeout) +// +// command = append(command, "--snapstore-temp-directory=/var/etcd/data/temp") +// command = append(command, "--enable-member-lease-renewal=true") +// +// heartbeatDuration := defaultHeartbeatDuration +// if etcd.Spec.Etcd.HeartbeatDuration != nil { +// heartbeatDuration = etcd.Spec.Etcd.HeartbeatDuration.Duration.String() +// } +// command = append(command, "--k8s-heartbeat-duration="+heartbeatDuration) +// +// if etcd.Spec.Backup.LeaderElection != nil { +// if etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout != nil { +// command = append(command, "--etcd-connection-timeout-leader-election="+etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout.Duration.String()) +// } +// +// if etcd.Spec.Backup.LeaderElection.ReelectionPeriod != nil { +// command = append(command, "--reelection-period="+etcd.Spec.Backup.LeaderElection.ReelectionPeriod.Duration.String()) +// } +// } +// +// return command +//} +// +//func getBackupPorts() []corev1.ContainerPort { +// return []corev1.ContainerPort{ +// { +// Name: "server", +// Protocol: "TCP", +// ContainerPort: defaultBackupPort, +// }, +// } +//} +// +//func getBackupResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { +// if etcd.Spec.Backup.Resources != nil { +// return *etcd.Spec.Backup.Resources +// } +// return defaultResourceRequirements +//} +// +//func getBackupRestoreEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { +// var ( +// env []corev1.EnvVar +// storageContainer string +// storeValues = etcd.Spec.Backup.Store +// ) +// +// if etcd.Spec.Backup.Store != nil { +// storageContainer = pointer.StringDeref(etcd.Spec.Backup.Store.Container, "") +// } +// +// // TODO(timuthy, shreyas-s-rao): Move STORAGE_CONTAINER a few lines below so that we can append and exit in one step. This should only be done in a release where a restart of etcd is acceptable. +// env = append(env, getEnvVarFromValue("STORAGE_CONTAINER", storageContainer)) +// env = append(env, getEnvVarFromField("POD_NAME", "metadata.name")) +// env = append(env, getEnvVarFromField("POD_NAMESPACE", "metadata.namespace")) +// +// if storeValues == nil { +// return env +// } +// +// provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) +// if err != nil { +// return env +// } +// +// // TODO(timuthy): move this to a non root path when we switch to a rootless distribution +// const credentialsMountPath = "/var/etcd-backup" +// switch provider { +// case utils.S3: +// env = append(env, getEnvVarFromValue("AWS_APPLICATION_CREDENTIALS", credentialsMountPath)) +// +// case utils.ABS: +// env = append(env, getEnvVarFromValue("AZURE_APPLICATION_CREDENTIALS", credentialsMountPath)) +// +// case utils.GCS: +// env = append(env, getEnvVarFromValue("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) +// +// case utils.Swift: +// env = append(env, getEnvVarFromValue("OPENSTACK_APPLICATION_CREDENTIALS", credentialsMountPath)) +// +// case utils.OSS: +// env = append(env, getEnvVarFromValue("ALICLOUD_APPLICATION_CREDENTIALS", credentialsMountPath)) +// +// case utils.ECS: +// env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) +// env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) +// env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) +// +// case utils.OCS: +// env = append(env, getEnvVarFromValue("OPENSHIFT_APPLICATION_CREDENTIALS", credentialsMountPath)) +// } +// +// return env +//} +// +//func getBackupRestoreVolumeMounts(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { +// volumeClaimTemplateName := etcd.Name +// if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { +// volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate +// } +// vms := []corev1.VolumeMount{ +// { +// Name: volumeClaimTemplateName, +// MountPath: "/var/etcd/data", +// }, +// { +// Name: "etcd-config-file", +// MountPath: "/var/etcd/config/", +// }, +// } +// +// vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) +// +// if etcd.Spec.Backup.Store == nil { +// return vms +// } +// +// provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) +// if err != nil { +// return vms +// } +// +// switch provider { +// case utils.Local: +// if etcd.Spec.Backup.Store.Container != nil { +// if useEtcdWrapper { +// vms = append(vms, corev1.VolumeMount{ +// Name: "host-storage", +// MountPath: "/home/nonroot/" + pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), +// }) +// } else { +// vms = append(vms, corev1.VolumeMount{ +// Name: "host-storage", +// MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), +// }) +// } +// } +// case utils.GCS: +// vms = append(vms, corev1.VolumeMount{ +// Name: "etcd-backup", +// MountPath: "/var/.gcp/", +// }) +// case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: +// vms = append(vms, corev1.VolumeMount{ +// Name: "etcd-backup", +// MountPath: "/var/etcd-backup/", +// }) +// } +// +// return vms +//} +// +//func getvolumeClaimTemplateName(etcd *druidv1alpha1.Etcd) string { +// volumeClaimTemplateName := etcd.Name +// if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { +// volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate +// } +// return volumeClaimTemplateName +//} +// +//func getStorageReq(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { +// storageCapacity := defaultStorageCapacity +// if etcd.Spec.StorageCapacity != nil { +// storageCapacity = *etcd.Spec.StorageCapacity +// } +// +// return corev1.ResourceRequirements{ +// Requests: corev1.ResourceList{ +// corev1.ResourceStorage: storageCapacity, +// }, +// } +//} +// +//func addEtcdContainer(etcdImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { +// return &corev1.Container{ +// Name: "etcd", +// Image: *etcdImage, +// ImagePullPolicy: corev1.PullIfNotPresent, +// Args: getEtcdCommandArgs(isEtcdWrapperEnabled, etcd), +// ReadinessProbe: &corev1.Probe{ +// ProbeHandler: getReadinessHandler(isEtcdWrapperEnabled, etcd), +// InitialDelaySeconds: 15, +// PeriodSeconds: 5, +// FailureThreshold: 5, +// }, +// Ports: getEtcdPorts(), +// Resources: getEtcdResources(etcd), +// Env: getEtcdEnvVars(etcd), +// VolumeMounts: getEtcdVolumeMounts(etcd), +// } +//} +// +//func addBackupRestoreContainer(etcdBackupImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { +// return &corev1.Container{ +// Name: "backup-restore", +// Image: *etcdBackupImage, +// ImagePullPolicy: corev1.PullIfNotPresent, +// Args: getBackupRestoreCommandArgs(etcd), +// Ports: getBackupPorts(), +// Resources: getBackupResources(etcd), +// Env: getBackupRestoreEnvVars(etcd), +// VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), +// SecurityContext: &corev1.SecurityContext{ +// Capabilities: &corev1.Capabilities{ +// Add: []corev1.Capability{ +// "SYS_PTRACE", +// }, +// }, +// }, +// } +//} +// +//func addInitContainersIfWrapperEnabled(initContainerImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) []corev1.Container { +// if !isEtcdWrapperEnabled { +// return []corev1.Container{} +// } +// +// // Initialize the slice with the 'change-permissions' container +// initContainers := []corev1.Container{ +// { +// Name: "change-permissions", +// Image: *initContainerImage, +// ImagePullPolicy: corev1.PullIfNotPresent, +// Command: []string{"sh", "-c", "--"}, +// Args: []string{"chown -R 65532:65532 /var/etcd/data"}, +// VolumeMounts: getEtcdVolumeMounts(etcd), +// SecurityContext: &corev1.SecurityContext{ +// RunAsGroup: pointer.Int64(0), +// RunAsNonRoot: pointer.Bool(false), +// RunAsUser: pointer.Int64(0), +// }, +// }, +// } +// +// if etcd.Spec.Backup.Store != nil { +// prov, _ := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) +// if prov == utils.Local { +// initContainers = append(initContainers, corev1.Container{ +// Name: "change-backup-bucket-permissions", +// Image: *initContainerImage, +// ImagePullPolicy: corev1.PullIfNotPresent, +// Command: []string{"sh", "-c", "--"}, +// Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, +// VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), +// SecurityContext: &corev1.SecurityContext{ +// RunAsGroup: pointer.Int64(0), +// RunAsNonRoot: pointer.Bool(false), +// RunAsUser: pointer.Int64(0), +// }, +// }) +// } +// } +// +// return initContainers +//} +// +//func getVolumeClaimTemplates(etcd *druidv1alpha1.Etcd) []corev1.PersistentVolumeClaim { +// return []corev1.PersistentVolumeClaim{{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: getvolumeClaimTemplateName(etcd), +// }, +// Spec: corev1.PersistentVolumeClaimSpec{ +// AccessModes: []corev1.PersistentVolumeAccessMode{ +// corev1.ReadWriteOnce, +// }, +// Resources: getStorageReq(etcd), +// StorageClassName: etcd.Spec.StorageClass, +// }, +// }, +// } +//} +// +//func addSecurityContextIfWrapperEnabled(isEtcdWrapperEnabled bool) *corev1.PodSecurityContext { +// return &corev1.PodSecurityContext{ +// RunAsGroup: pointer.Int64(65532), +// RunAsNonRoot: pointer.Bool(true), +// RunAsUser: pointer.Int64(65532), +// } +//} +// +//func getPriorityClassName(etcd *druidv1alpha1.Etcd) string { +// if etcd.Spec.PriorityClassName != nil { +// return *etcd.Spec.PriorityClassName +// } +// return "" +//} +// +//// hasImmutableFieldChanged checks if any immutable fields have changed in the StatefulSet +//// specification compared to the Etcd object. It returns true if there are changes in the immutable fields. +//// Currently, it checks for changes in the ServiceName and PodManagementPolicy. +//func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { +// return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement +//} +// +//func getVolumes(client client.Client, logger logr.Logger, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { +// vs := []corev1.Volume{ +// { +// Name: "etcd-config-file", +// VolumeSource: corev1.VolumeSource{ +// ConfigMap: &corev1.ConfigMapVolumeSource{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: etcd.GetConfigMapName(), +// }, +// Items: []corev1.KeyToPath{ +// { +// Key: "etcd.conf.yaml", +// Path: "etcd.conf.yaml", +// }, +// }, +// DefaultMode: pointer.Int32(0644), +// }, +// }, +// }, +// } +// +// if etcd.Spec.Etcd.ClientUrlTLS != nil { +// vs = append(vs, corev1.Volume{ +// Name: "client-url-ca-etcd", +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, +// }, +// }, +// }, +// corev1.Volume{ +// Name: "client-url-etcd-server-tls", +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, +// }, +// }, +// }, +// corev1.Volume{ +// Name: "client-url-etcd-client-tls", +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, +// }, +// }, +// }) +// } +// +// if etcd.Spec.Etcd.PeerUrlTLS != nil { +// vs = append(vs, corev1.Volume{ +// Name: "peer-url-ca-etcd", +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, +// }, +// }, +// }, +// corev1.Volume{ +// Name: "peer-url-etcd-server-tls", +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, +// }, +// }, +// }) +// } +// +// if etcd.Spec.Backup.Store == nil { +// return vs, nil +// } +// +// storeValues := etcd.Spec.Backup.Store +// provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) +// if err != nil { +// return vs, nil +// } +// +// switch provider { +// case "Local": +// hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, client, logger, storeValues, etcd.GetNamespace()) +// if err != nil { +// return nil, err +// } +// +// hpt := corev1.HostPathDirectory +// vs = append(vs, corev1.Volume{ +// Name: "host-storage", +// VolumeSource: corev1.VolumeSource{ +// HostPath: &corev1.HostPathVolumeSource{ +// Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), +// Type: &hpt, +// }, +// }, +// }) +// case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: +// if storeValues.SecretRef == nil { +// return nil, fmt.Errorf("no secretRef configured for backup store") +// } +// +// vs = append(vs, corev1.Volume{ +// Name: "etcd-backup", +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: storeValues.SecretRef.Name, +// }, +// }, +// }) +// } +// +// return vs, nil +//} +// +//func getConfigMapChecksum(cl client.Client, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (string, error) { +// cm := &corev1.ConfigMap{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: etcd.GetConfigMapName(), +// Namespace: etcd.Namespace, +// }, +// } +// if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { +// +// return "", err +// } +// jsonString, err := json.Marshal(cm.Data) +// if err != nil { +// return "", err +// } +// +// return gardenerutils.ComputeSHA256Hex(jsonString), nil +//} +// +//func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger logr.Logger, opName string, sts *appsv1.StatefulSet) error { +// // Get all Pods belonging to the StatefulSet +// podList := &corev1.PodList{} +// listOpts := []client.ListOption{ +// client.InNamespace(sts.Namespace), +// client.MatchingLabels(sts.Spec.Selector.MatchLabels), +// } +// +// if err := cl.List(ctx, podList, listOpts...); err != nil { +// logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) +// return err +// } +// +// for _, pod := range podList.Items { +// if err := cl.Delete(ctx, &pod); err != nil { +// logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) +// return err +// } +// } +// +// return nil +//} +// +//// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled +//func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { +// return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil +//} diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 1d231684e..d14e3123e 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -5,19 +5,17 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/features" + corev1 "k8s.io/api/core/v1" "k8s.io/component-base/featuregate" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - druidutils "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -49,7 +47,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } if existingSts == nil { - return r.doCreateOrPatch(ctx, etcd, etcd.Spec.Replicas) + return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) } if err := r.handlePeerTLSEnabled(ctx, etcd, existingSts); err != nil { @@ -60,15 +58,15 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) return fmt.Errorf("error handling immutable field updates: %w", err) } - return r.doCreateOrPatch(ctx, etcd, etcd.Spec.Replicas) + return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return client.IgnoreNotFound(r.client.Delete(ctx, emptyStatefulset(getObjectKey(etcd)))) + return client.IgnoreNotFound(r.client.Delete(ctx, emptyStatefulSet(getObjectKey(etcd)))) } func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { - sts := emptyStatefulset(getObjectKey(etcd)) + sts := emptyStatefulSet(getObjectKey(etcd)) if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { if errors.IsNotFound(err) { return nil, nil @@ -78,59 +76,15 @@ func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *dr return sts, nil } -func (r _resource) doCreateOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { - desiredStatefulSet := emptyStatefulset(getObjectKey(etcd)) - etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.useEtcdWrapper) - if err != nil { - return err - } - podVolumes, err := getVolumes(r.client, r.logger, ctx, etcd) - if err != nil { - return err - } - - configMapChecksum, err := getConfigMapChecksum(r.client, ctx, etcd) - if err != nil { - return err - } - +func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { + desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) mutatingFn := func() error { - desiredStatefulSet.ObjectMeta = extractObjectMetaFromEtcd(etcd) - desiredStatefulSet.Spec = appsv1.StatefulSetSpec{ - Replicas: &replicas, - ServiceName: etcd.GetPeerServiceName(), - Selector: &metav1.LabelSelector{ - MatchLabels: etcd.GetDefaultLabels(), - }, - PodManagementPolicy: appsv1.ParallelPodManagement, - UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: appsv1.RollingUpdateStatefulSetStrategyType, - }, - VolumeClaimTemplates: getVolumeClaimTemplates(etcd), - Template: corev1.PodTemplateSpec{ - ObjectMeta: extractPodObjectMetaFromEtcd(etcd, configMapChecksum), - Spec: corev1.PodSpec{ - HostAliases: []corev1.HostAlias{{ - IP: "127.0.0.1", - Hostnames: []string{etcd.Name + "-local"}}}, - ServiceAccountName: etcd.GetServiceAccountName(), - Affinity: etcd.Spec.SchedulingConstraints.Affinity, - TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, - Containers: []corev1.Container{ - *addEtcdContainer(etcdImage, r.useEtcdWrapper, etcd), - *addBackupRestoreContainer(etcdBackupImage, r.useEtcdWrapper, etcd), - }, - InitContainers: addInitContainersIfWrapperEnabled(initContainerImage, r.useEtcdWrapper, etcd), - ShareProcessNamespace: pointer.Bool(true), - Volumes: podVolumes, - SecurityContext: addSecurityContextIfWrapperEnabled(r.useEtcdWrapper), - PriorityClassName: getPriorityClassName(etcd), - }, - }, + if builder, err := newStsBuilder(r.client, r.logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { + return err + } else { + return builder.Build(ctx) } - return nil } - opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) if err != nil { return err @@ -140,6 +94,68 @@ func (r _resource) doCreateOrPatch(ctx resource.OperatorContext, etcd *druidv1al return nil } +//func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { +// desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) +// etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.useEtcdWrapper) +// if err != nil { +// return err +// } +// podVolumes, err := getVolumes(r.client, r.logger, ctx, etcd) +// if err != nil { +// return err +// } +// +// configMapChecksum, err := getConfigMapChecksum(r.client, ctx, etcd) +// if err != nil { +// return err +// } +// +// mutatingFn := func() error { +// desiredStatefulSet.ObjectMeta = extractObjectMetaFromEtcd(etcd) +// desiredStatefulSet.Spec = appsv1.StatefulSetSpec{ +// Replicas: &replicas, +// ServiceName: etcd.GetPeerServiceName(), +// Selector: &metav1.LabelSelector{ +// MatchLabels: etcd.GetDefaultLabels(), +// }, +// PodManagementPolicy: appsv1.ParallelPodManagement, +// UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ +// Type: appsv1.RollingUpdateStatefulSetStrategyType, +// }, +// VolumeClaimTemplates: getVolumeClaimTemplates(etcd), +// Template: corev1.PodTemplateSpec{ +// ObjectMeta: extractPodObjectMetaFromEtcd(etcd, configMapChecksum), +// Spec: corev1.PodSpec{ +// HostAliases: []corev1.HostAlias{{ +// IP: "127.0.0.1", +// Hostnames: []string{etcd.Name + "-local"}}}, +// ServiceAccountName: etcd.GetServiceAccountName(), +// Affinity: etcd.Spec.SchedulingConstraints.Affinity, +// TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, +// Containers: []corev1.Container{ +// *addEtcdContainer(etcdImage, r.useEtcdWrapper, etcd), +// *addBackupRestoreContainer(etcdBackupImage, r.useEtcdWrapper, etcd), +// }, +// InitContainers: addInitContainersIfWrapperEnabled(initContainerImage, r.useEtcdWrapper, etcd), +// ShareProcessNamespace: pointer.Bool(true), +// Volumes: podVolumes, +// SecurityContext: addSecurityContextIfWrapperEnabled(r.useEtcdWrapper), +// PriorityClassName: getPriorityClassName(etcd), +// }, +// }, +// } +// return nil +// } +// +// opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) +// if err != nil { +// return err +// } +// +// r.logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) +// return nil +//} + func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { if existingSts.Generation > 1 && hasImmutableFieldChanged(existingSts, etcd) { r.logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") @@ -151,6 +167,13 @@ func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etc return nil } +// hasImmutableFieldChanged checks if any immutable fields have changed in the StatefulSet +// specification compared to the Etcd object. It returns true if there are changes in the immutable fields. +// Currently, it checks for changes in the ServiceName and PodManagementPolicy. +func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { + return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement +} + func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) if err != nil { @@ -160,7 +183,7 @@ func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *drui if isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { r.logger.Info("Enabling TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) - if err := r.doCreateOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { + if err := r.createOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %v", err) } @@ -182,7 +205,35 @@ func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *drui return nil } -func emptyStatefulset(objectKey client.ObjectKey) *appsv1.StatefulSet { +// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled +func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { + return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil +} + +func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger logr.Logger, opName string, sts *appsv1.StatefulSet) error { + // Get all Pods belonging to the StatefulSet + podList := &corev1.PodList{} + listOpts := []client.ListOption{ + client.InNamespace(sts.Namespace), + client.MatchingLabels(sts.Spec.Selector.MatchLabels), + } + + if err := cl.List(ctx, podList, listOpts...); err != nil { + logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) + return err + } + + for _, pod := range podList.Items { + if err := cl.Delete(ctx, &pod); err != nil { + logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) + return err + } + } + + return nil +} + +func emptyStatefulSet(objectKey client.ObjectKey) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: objectKey.Name, diff --git a/internal/utils/image.go b/internal/utils/image.go index 8014c7bc7..1d9996e59 100755 --- a/internal/utils/image.go +++ b/internal/utils/image.go @@ -39,22 +39,22 @@ func getEtcdImageKeys(useEtcdWrapper bool) (etcdImageKey string, etcdbrImageKey // It will give preference to images that are set in the etcd spec and only if the image is not found in it should // it be picked up from the image vector if it's set there. // A return value of nil for either of the images indicates that the image is not set. -func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcdWrapper bool) (*string, *string, *string, error) { +func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcdWrapper bool) (string, string, string, error) { etcdImageKey, etcdbrImageKey, initContainerImageKey := getEtcdImageKeys(useEtcdWrapper) etcdImage, err := chooseImage(etcdImageKey, etcd.Spec.Etcd.Image, iv) if err != nil { - return nil, nil, nil, err + return "", "", "", err } etcdBackupRestoreImage, err := chooseImage(etcdbrImageKey, etcd.Spec.Backup.Image, iv) if err != nil { - return nil, nil, nil, err + return "", "", "", err } initContainerImage, err := chooseImage(initContainerImageKey, nil, iv) if err != nil { - return nil, nil, nil, err + return "", "", "", err } - return etcdImage, etcdBackupRestoreImage, initContainerImage, nil + return *etcdImage, *etcdBackupRestoreImage, *initContainerImage, nil } // chooseImage selects an image based on the given key, specImage, and image vector. diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index 5077ea04a..8410522f8 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -100,3 +100,19 @@ func IsEmptyString(s string) bool { } return false } + +// IfConditionOr implements a simple ternary operator, if the passed condition is true then trueVal is returned else falseVal is returned. +func IfConditionOr[T any](condition bool, trueVal, falseVal T) T { + if condition { + return trueVal + } + return falseVal +} + +// IsNilOrEmptyStringPtr returns true if the string pointer is nil or the return value of IsEmptyString(s). +func IsNilOrEmptyStringPtr(s *string) bool { + if s == nil { + return true + } + return IsEmptyString(*s) +} From 32641f5c82eca11705515ff6ee2018f5f9f99f6a Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Wed, 6 Dec 2023 11:17:27 +0530 Subject: [PATCH 024/235] Fix incorrect scheme used in ListenClientUrls for etcd config --- internal/operator/configmap/etcdconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index 367dd4af4..98fadda01 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -77,7 +77,7 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionMode: utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), AutoCompactionRetention: utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), - ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), + ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), } From 467183aa0cce377905857a65cd0a399f92835c45 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Wed, 6 Dec 2023 11:20:45 +0530 Subject: [PATCH 025/235] Update MemberLease labels to include etcd name-specific labels --- internal/operator/memberlease/memberlease.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 7d626664a..9ab01ad83 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -68,7 +68,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) doCreateOrUpdate(ctx context.Context, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { lease := emptyMemberLease(objKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { - lease.Labels = etcd.GetDefaultLabels() + lease.Labels = utils.MergeMaps[string](utils.GetMemberLeaseLabels(etcd.Name), etcd.GetDefaultLabels()) lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} return nil }) From a040560e66c4e82f0405225177d09ef74a8b30cb Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 7 Dec 2023 10:21:47 +0530 Subject: [PATCH 026/235] added godoc for errors.go --- internal/errors/errors.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/errors/errors.go b/internal/errors/errors.go index d951b9ec1..681994871 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -9,6 +9,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// DruidError is a custom error that should be used throughout druid which encapsulates +// the underline error (cause) along with error code and contextual information captured +// as operation during which an error occurred and any custom message. type DruidError struct { Code druidv1alpha1.ErrorCode Cause error @@ -20,6 +23,8 @@ func (r *DruidError) Error() string { return fmt.Sprintf("[Operation: %s, Code: %s] %s", r.Operation, r.Code, r.Cause.Error()) } +// WrapError wraps an error and contextual information like code, operation and message to create a DruidError +// Consumers can use errors.As or errors.Is to check if the error is of type DruidError and get its constituent fields. func WrapError(err error, code druidv1alpha1.ErrorCode, operation string, message string) error { if err == nil { return nil @@ -32,6 +37,8 @@ func WrapError(err error, code druidv1alpha1.ErrorCode, operation string, messag } } +// MapToLastErrors maps a slice of DruidError's and maps these to druidv1alpha1.LastError slice which +// will be set in the status of an etcd resource. func MapToLastErrors(errs []error) []druidv1alpha1.LastError { lastErrs := make([]druidv1alpha1.LastError, 0, len(errs)) for _, err := range errs { From 1c3d1585d6e7ea9cca68ef6acf8b234c1335f976 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 7 Dec 2023 10:25:56 +0530 Subject: [PATCH 027/235] added missing exported field godoc --- internal/errors/errors.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 681994871..ecdb77a76 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -13,10 +13,14 @@ import ( // the underline error (cause) along with error code and contextual information captured // as operation during which an error occurred and any custom message. type DruidError struct { - Code druidv1alpha1.ErrorCode - Cause error + // Code indicates the category of error adding contextual information to the underline error. + Code druidv1alpha1.ErrorCode + // Cause is the underline error. + Cause error + // Operation is the semantic operation during which this error is created/wrapped. Operation string - Message string + // Message is the custom message providing additional context for the error. + Message string } func (r *DruidError) Error() string { From 7a94aaaf7e2db5727dc457cb9c0f7213114baede Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 26 Dec 2023 08:29:47 +0530 Subject: [PATCH 028/235] WIP commit - refactored sts resource operator --- internal/operator/statefulset/builder.go | 141 ++++++++++++++++++++--- 1 file changed, 124 insertions(+), 17 deletions(-) diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 52780b26b..6b480304d 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -61,6 +61,10 @@ type stsBuilder struct { initContainerImage string sts *appsv1.StatefulSet logger logr.Logger + + clientPort int32 + serverPort int32 + backupPort int32 } func newStsBuilder(client client.Client, @@ -89,6 +93,9 @@ func newStsBuilder(client client.Client, etcdBackupRestoreImage: etcdBackupRestoreImage, initContainerImage: initContainerImage, sts: sts, + clientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, defaultClientPort), + serverPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort), + backupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, defaultBackupPort), }, nil } @@ -288,7 +295,7 @@ func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) return corev1.VolumeMount{ Name: volumeClaimTemplateName, - MountPath: "/var/etcd/data/", + MountPath: "/var/etcd/data", } } @@ -302,13 +309,13 @@ func (b *stsBuilder) getEtcdContainer() corev1.Container { Ports: []corev1.ContainerPort{ { Name: "server", - Protocol: "TCP", - ContainerPort: defaultServerPort, + Protocol: corev1.ProtocolTCP, + ContainerPort: b.serverPort, }, { Name: "client", - Protocol: "TCP", - ContainerPort: defaultClientPort, + Protocol: corev1.ProtocolTCP, + ContainerPort: b.clientPort, }, }, Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Etcd.Resources, defaultResourceRequirements), @@ -322,11 +329,24 @@ func (b *stsBuilder) getBackupRestoreContainer() corev1.Container { Name: "backup-restore", Image: b.etcdBackupRestoreImage, ImagePullPolicy: corev1.PullIfNotPresent, - Args: nil, - Ports: nil, - Env: nil, - Resources: corev1.ResourceRequirements{}, - VolumeMounts: nil, + Args: b.getBackupRestoreContainerCommandArgs(), + Ports: []corev1.ContainerPort{ + { + Name: "server", + Protocol: corev1.ProtocolTCP, + ContainerPort: b.backupPort, + }, + }, + Env: b.getBackupRestorContainerEnvVars(), + Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Backup.Resources, defaultResourceRequirements), + VolumeMounts: b.getBackupRestoreContainerVolumeMounts(), + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "SYS_PTRACE", + }, + }, + }, } } @@ -374,13 +394,13 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { commandArgs = append(commandArgs, fmt.Sprintf("--cacert=/var/etcd/ssl/client/ca/%s", dataKey)) commandArgs = append(commandArgs, "--insecure-transport=false") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=false") - commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, defaultClientPort)) - commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=https://%s:%d", b.etcd.GetClientServiceName(), defaultClientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=https://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) } else { commandArgs = append(commandArgs, "--insecure-transport=true") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=true") - commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=http://%s-local:%d", b.etcd.Name, defaultClientPort)) - commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), defaultClientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=http://%s-local:%d", b.etcd.Name, b.clientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) } if b.etcd.Spec.Backup.TLS != nil { commandArgs = append(commandArgs, "--server-cert=/var/etcd/ssl/client/server/tls.crt") @@ -391,7 +411,9 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { // ----------------------------------------------------------------------------------------------------------------- commandArgs = append(commandArgs, "--data-dir=/var/etcd/data/new.etcd") commandArgs = append(commandArgs, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") + commandArgs = append(commandArgs, "--snapstore-temp-directory=/var/etcd/data/temp") commandArgs = append(commandArgs, fmt.Sprintf("--etcd-connection-timeout=%s", defaultEtcdConnectionTimeout)) + commandArgs = append(commandArgs, "--enable-member-lease-renewal=true") var quota = defaultQuota if b.etcd.Spec.Etcd.Quota != nil { @@ -522,9 +544,9 @@ func (b *stsBuilder) getEtcdContainerReadinessProbeCommand() []string { cmdBuilder.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) cmdBuilder.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") cmdBuilder.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") - cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", b.etcd.Name, defaultClientPort)) + cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) } else { - cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", b.etcd.Name, defaultClientPort)) + cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", b.etcd.Name, b.clientPort)) } cmdBuilder.WriteString(" get foo") cmdBuilder.WriteString(" --consistency=l") @@ -563,7 +585,7 @@ func (b *stsBuilder) getEtcdContainerEnvVars() []corev1.EnvVar { } backTLSEnabled := b.etcd.Spec.Backup.TLS != nil scheme := utils.IfConditionOr[string](backTLSEnabled, "https", "http") - endpoint := fmt.Sprintf("%s://%s-local:%d", scheme, b.etcd.Name, defaultBackupPort) + endpoint := fmt.Sprintf("%s://%s-local:%d", scheme, b.etcd.Name, b.backupPort) return []corev1.EnvVar{ {Name: "ENABLE_TLS", Value: strconv.FormatBool(backTLSEnabled)}, @@ -571,6 +593,59 @@ func (b *stsBuilder) getEtcdContainerEnvVars() []corev1.EnvVar { } } +func (b *stsBuilder) getBackupRestorContainerEnvVars() []corev1.EnvVar { + var ( + env []corev1.EnvVar + storageContainer string + storeValues = b.etcd.Spec.Backup.Store + ) + + if b.etcd.Spec.Backup.Store != nil { + storageContainer = pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, "") + } + + env = append(env, getEnvVarFromField("POD_NAME", "metadata.name")) + env = append(env, getEnvVarFromField("POD_NAMESPACE", "metadata.namespace")) + + if storeValues == nil { + return env + } + + provider, err := utils.StorageProviderFromInfraProvider(b.etcd.Spec.Backup.Store.Provider) + if err != nil { + return env + } + env = append(env, getEnvVarFromValue("STORAGE_CONTAINER", storageContainer)) + + const credentialsMountPath = "/var/etcd-backup" + switch provider { + case utils.S3: + env = append(env, getEnvVarFromValue("AWS_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.ABS: + env = append(env, getEnvVarFromValue("AZURE_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.GCS: + env = append(env, getEnvVarFromValue("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) + + case utils.Swift: + env = append(env, getEnvVarFromValue("OPENSTACK_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.OSS: + env = append(env, getEnvVarFromValue("ALICLOUD_APPLICATION_CREDENTIALS", credentialsMountPath)) + + case utils.ECS: + env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) + env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) + env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) + + case utils.OCS: + env = append(env, getEnvVarFromValue("OPENSHIFT_APPLICATION_CREDENTIALS", credentialsMountPath)) + } + + return env +} + func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { if !b.useEtcdWrapper { return nil @@ -749,3 +824,35 @@ func getBackupStoreProvider(etcd *druidv1alpha1.Etcd) (*string, error) { } return &provider, nil } + +func getEnvVarFromValue(name, value string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + Value: value, + } +} + +func getEnvVarFromField(name, fieldPath string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: fieldPath, + }, + }, + } +} + +func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + Key: secretKey, + }, + }, + } +} From c78e56ec6f4bd319f27c67c0dcf3d4d6671deb87 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 26 Dec 2023 08:30:12 +0530 Subject: [PATCH 029/235] Removed unused function --- internal/operator/peerservice/peerservice.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index e94361e6a..1eb5727e6 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -64,15 +64,6 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace} } -func (r _resource) getLabels(etcd *druidv1alpha1.Etcd) map[string]string { - var labelMaps []map[string]string - labelMaps = append(labelMaps, etcd.GetDefaultLabels()) - if etcd.Spec.Etcd.ClientService != nil { - labelMaps = append(labelMaps, etcd.Spec.Etcd.ClientService.Labels) - } - return utils.MergeMaps[string, string](labelMaps...) -} - func emptyPeerService(objectKey client.ObjectKey) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ From b9930b8df17a88f7bb56a20198cb38f2dccafb86 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 26 Dec 2023 08:31:25 +0530 Subject: [PATCH 030/235] Add unit test for peerservice --- .../operator/peerservice/peerservice_test.go | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 internal/operator/peerservice/peerservice_test.go diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go new file mode 100644 index 000000000..0af2f617b --- /dev/null +++ b/internal/operator/peerservice/peerservice_test.go @@ -0,0 +1,237 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package peerservice + +import ( + "context" + "fmt" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames_WithExistingService(t *testing.T) { + g, ctx, etcd, _, op := setupWithFakeClient(t, true) + + err := op.Sync(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + names, err := op.GetExistingResourceNames(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(names).To(gomega.ContainElement(etcd.GetPeerServiceName())) + +} + +func TestGetExistingResourceNames_ServiceNotFound(t *testing.T) { + g, ctx, etcd, _, op := setupWithFakeClient(t, false) + + names, err := op.GetExistingResourceNames(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(names).To(gomega.BeEmpty()) +} + +func TestGetExistingResourceNames_WithError(t *testing.T) { + g, ctx, etcd, cl, op := setupWithMockClient(t) + cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() + + names, err := op.GetExistingResourceNames(ctx, etcd) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(names).To(gomega.BeEmpty()) +} + +// ----------------------------------- Sync ----------------------------------- + +func TestSync_CreateNewService(t *testing.T) { + g, ctx, etcd, cl, op := setupWithFakeClient(t, false) + err := op.Sync(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetPeerServiceName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) + g.Expect(err).NotTo(gomega.HaveOccurred()) + checkPeerService(g, service, etcd) + +} + +func TestSync_UpdateExistingService(t *testing.T) { + g, ctx, etcd, cl, op := setupWithFakeClient(t, true) + etcd.Spec.Etcd.ServerPort = nil + + err := op.Sync(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetPeerServiceName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(service).ToNot(gomega.BeNil()) + checkPeerService(g, service, etcd) +} + +func TestSync_WithClientError(t *testing.T) { + g, ctx, etcd, cl, op := setupWithMockClient(t) + cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() + cl.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake create error")).AnyTimes() + cl.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake update error")).AnyTimes() + + err := op.Sync(ctx, etcd) + g.Expect(err).To(gomega.HaveOccurred()) +} + +// ----------------------------- TriggerDelete ------------------------------- + +func TestTriggerDelete_ExistingService(t *testing.T) { + g, ctx, etcd, cl, op := setupWithFakeClient(t, true) + err := op.TriggerDelete(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + serviceList := &corev1.List{} + err = cl.List(ctx, serviceList) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(serviceList.Items).To(gomega.BeEmpty()) +} + +func TestTriggerDelete_ServiceNotFound(t *testing.T) { + g, ctx, etcd, _, op := setupWithFakeClient(t, false) + + err := op.TriggerDelete(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func TestTriggerDelete_WithClientError(t *testing.T) { + g, ctx, etcd, cl, op := setupWithMockClient(t) + // Configure the mock client to return an error on delete operation + cl.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake delete error")).AnyTimes() + + err := op.TriggerDelete(ctx, etcd) + g.Expect(err).To(gomega.HaveOccurred()) +} + +// ---------------------------- Helper Functions ----------------------------- + +func sampleEtcd() *druidv1alpha1.Etcd { + return &druidv1alpha1.Etcd{ + + ObjectMeta: metav1.ObjectMeta{ + Name: "test-etcd", + Namespace: "test-namespace", + UID: "xxx-yyy-zzz", + }, + Spec: druidv1alpha1.EtcdSpec{ + + Backup: druidv1alpha1.BackupSpec{ + Port: pointer.Int32(1111), + }, + Etcd: druidv1alpha1.EtcdConfig{ + ClientPort: pointer.Int32(2222), + ServerPort: pointer.Int32(3333), + }, + }, + } +} + +func checkPeerService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(svc.Labels).To(gomega.Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.PublishNotReadyAddresses).To(gomega.BeTrue()) + g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) + g.Expect(svc.Spec.ClusterIP).To(gomega.Equal(corev1.ClusterIPNone)) + g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) + g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( + gomega.Equal(corev1.ServicePort{ + Name: "peer", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }), + )) +} + +func existingService(etcd *druidv1alpha1.Etcd) *corev1.Service { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetPeerServiceName(), + Namespace: etcd.Namespace, + Labels: etcd.GetDefaultLabels(), + OwnerReferences: []metav1.OwnerReference{ + etcd.GetAsOwnerReference(), + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, + SessionAffinity: corev1.ServiceAffinityNone, + Selector: etcd.GetDefaultLabels(), + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "peer", + Protocol: corev1.ProtocolTCP, + Port: *etcd.Spec.Etcd.ServerPort, + TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), + }, + }, + }, + } + return service +} + +func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl client.Client, op resource.Operator) { + g = gomega.NewWithT(t) + ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") + etcd = sampleEtcd() + if withExistingService { + cl = fakeclient.NewClientBuilder().WithObjects(existingService(etcd)).Build() + } else { + + cl = fakeclient.NewClientBuilder().Build() + } + op = New(cl, log.Log.WithName("test")) + return +} + +func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl *mockclient.MockClient, op resource.Operator) { + g = gomega.NewWithT(t) + ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") + etcd = sampleEtcd() + cl = mockclient.NewMockClient(gomock.NewController(t)) + op = New(cl, log.Log.WithName("test")) + return +} From 63b8aaa7f3ca80eb7545afa05daf7d4ee9c6a6cd Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 26 Dec 2023 08:32:04 +0530 Subject: [PATCH 031/235] Add unit test for clientservice --- .../clientservice/clientservice_test.go | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 internal/operator/clientservice/clientservice_test.go diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go new file mode 100644 index 000000000..0bc94483b --- /dev/null +++ b/internal/operator/clientservice/clientservice_test.go @@ -0,0 +1,283 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientservice + +import ( + "context" + "fmt" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// ------------------------ GetExistingResourceNames ------------------------ + +func TestClientServiceGetExistingResourceNames_WithExistingService(t *testing.T) { + g, ctx, etcd, _, op := setupWithFakeClient(t, true) + + err := op.Sync(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + names, err := op.GetExistingResourceNames(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(names).To(gomega.ContainElement(etcd.GetClientServiceName())) +} + +func TestClientServiceGetExistingResourceNames_ServiceNotFound(t *testing.T) { + g, ctx, etcd, _, op := setupWithFakeClient(t, false) + + names, err := op.GetExistingResourceNames(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(names).To(gomega.BeEmpty()) +} + +func TestClientServiceGetExistingResourceNames_WithError(t *testing.T) { + g, ctx, etcd, cl, op := setupWithMockClient(t) + cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() + + names, err := op.GetExistingResourceNames(ctx, etcd) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(names).To(gomega.BeEmpty()) +} + +// ----------------------------------- Sync ----------------------------------- + +func TestClientServiceSync_CreateNewService(t *testing.T) { + g, ctx, etcd, cl, op := setupWithFakeClient(t, false) + err := op.Sync(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetClientServiceName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) + g.Expect(err).NotTo(gomega.HaveOccurred()) + checkClientService(g, service, etcd) +} + +func TestClientServiceSync_UpdateExistingService(t *testing.T) { + g, ctx, etcd, cl, op := setupWithFakeClient(t, true) + etcd.Spec.Etcd.ServerPort = nil + etcd.Spec.Etcd.ClientPort = nil + etcd.Spec.Backup.Port = nil + + etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{ + Labels: map[string]string{"testingKey": "testingValue"}, + Annotations: map[string]string{"testingAnnotationKey": "testingValue"}, + } + + err := op.Sync(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetClientServiceName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(service).ToNot(gomega.BeNil()) + checkClientService(g, service, etcd) +} + +func TestClientServiceSync_WithClientError(t *testing.T) { + g, ctx, etcd, cl, op := setupWithMockClient(t) + cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() + cl.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake create error")).AnyTimes() + cl.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake update error")).AnyTimes() + + err := op.Sync(ctx, etcd) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrSyncingClientService))) +} + +// ----------------------------- TriggerDelete ------------------------------- + +func TestClientServiceTriggerDelete_ExistingService(t *testing.T) { + g, ctx, etcd, cl, op := setupWithFakeClient(t, true) + err := op.TriggerDelete(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + serviceList := &corev1.List{} + err = cl.List(ctx, serviceList) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(serviceList.Items).To(gomega.BeEmpty()) +} + +func TestClientServiceTriggerDelete_ServiceNotFound(t *testing.T) { + g, ctx, etcd, _, op := setupWithFakeClient(t, false) + + err := op.TriggerDelete(ctx, etcd) + g.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func TestClientServiceTriggerDelete_WithClientError(t *testing.T) { + g, ctx, etcd, cl, op := setupWithMockClient(t) + // Configure the mock client to return an error on delete operation + cl.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake delete error")).AnyTimes() + + err := op.TriggerDelete(ctx, etcd) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrDeletingClientService))) +} + +// ---------------------------- Helper Functions ----------------------------- + +func sampleEtcd() *druidv1alpha1.Etcd { + return &druidv1alpha1.Etcd{ + + ObjectMeta: metav1.ObjectMeta{ + Name: "test-etcd", + Namespace: "test-namespace", + UID: "xxx-yyy-zzz", + }, + Spec: druidv1alpha1.EtcdSpec{ + + Backup: druidv1alpha1.BackupSpec{ + Port: pointer.Int32(1111), + }, + Etcd: druidv1alpha1.EtcdConfig{ + ClientPort: pointer.Int32(2222), + ServerPort: pointer.Int32(3333), + }, + }, + } +} + +func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + + expectedLabels := etcd.GetDefaultLabels() + var expectedAnnotations map[string]string + if etcd.Spec.Etcd.ClientService != nil { + expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations + expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) + } + g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(svc.Annotations).To(gomega.Equal(expectedAnnotations)) + g.Expect(svc.Labels).To(gomega.Equal(expectedLabels)) + g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) + g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) + g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( + gomega.Equal(corev1.ServicePort{ + Name: "client", + Protocol: corev1.ProtocolTCP, + Port: clientPort, + TargetPort: intstr.FromInt(int(clientPort)), + }), + gomega.Equal(corev1.ServicePort{ + Name: "server", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }), + gomega.Equal(corev1.ServicePort{ + Name: "backuprestore", + Protocol: corev1.ProtocolTCP, + Port: backupPort, + TargetPort: intstr.FromInt(int(backupPort)), + }), + )) +} + +func existingClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { + var annotations, labels map[string]string + if etcd.Spec.Etcd.ClientService != nil { + annotations = etcd.Spec.Etcd.ClientService.Annotations + labels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) + + } + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetClientServiceName(), + Namespace: etcd.Namespace, + Annotations: annotations, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{ + etcd.GetAsOwnerReference(), + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, + SessionAffinity: corev1.ServiceAffinityNone, + Selector: etcd.GetDefaultLabels(), + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "client", + Protocol: corev1.ProtocolTCP, + Port: *etcd.Spec.Etcd.ClientPort, + TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ClientPort)), + }, + { + Name: "server", + Protocol: corev1.ProtocolTCP, + Port: *etcd.Spec.Etcd.ServerPort, + TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), + }, + { + Name: "backuprestore", + Protocol: corev1.ProtocolTCP, + Port: *etcd.Spec.Backup.Port, + TargetPort: intstr.FromInt(int(*etcd.Spec.Backup.Port)), + }, + }, + }, + } +} + +func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl client.Client, op resource.Operator) { + g = gomega.NewWithT(t) + ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") + etcd = sampleEtcd() + if withExistingService { + cl = fakeclient.NewClientBuilder().WithObjects(existingClientService(etcd)).Build() + } else { + + cl = fakeclient.NewClientBuilder().Build() + } + op = New(cl, log.Log.WithName("test")) + return +} + +func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl *mockclient.MockClient, op resource.Operator) { + g = gomega.NewWithT(t) + ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") + etcd = sampleEtcd() + cl = mockclient.NewMockClient(gomock.NewController(t)) + op = New(cl, log.Log.WithName("test")) + return +} From a204e033e4c328da9a521ee5ca4542f46281d6a5 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 26 Dec 2023 08:34:42 +0530 Subject: [PATCH 032/235] Add StatefulSet nil check in dataVolumesReady condition --- internal/health/condition/check_data_volumes_ready.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go index 63f9614f4..a72a7a89c 100644 --- a/internal/health/condition/check_data_volumes_ready.go +++ b/internal/health/condition/check_data_volumes_ready.go @@ -17,6 +17,7 @@ package condition import ( "context" "fmt" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -42,6 +43,12 @@ func (d *dataVolumesReady) Check(ctx context.Context, etcd druidv1alpha1.Etcd) R return res } + if sts == nil { + res.reason = "StatefulSetNotFound" + res.message = fmt.Sprintf("StatefulSet: %q not found for etcd", etcd.Name) + return res + } + pvcEvents, err := utils.FetchPVCWarningEventsForStatefulSet(ctx, d.cl, sts) if err != nil { res.reason = "UnableToFetchWarningEventsForDataVolumes" From f7ccfa6f248b17b8007e5a1d05f28aa00aeca1e1 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 26 Dec 2023 08:38:21 +0530 Subject: [PATCH 033/235] Implement RetryOnConflict in etcd Status Updates for LastOperation --- internal/controller/utils/etcdstatus.go | 38 ++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index 978b7c067..5949a9f3a 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -8,6 +8,7 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -69,23 +70,28 @@ func (l *lastOpErrRecorder) RecordError(ctx resource.OperatorContext, etcd *drui } func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, operationState druidv1alpha1.LastOperationState, description string, lastErrors ...druidv1alpha1.LastError) error { - etcdPatch := client.StrategicMergeFrom(etcd.DeepCopy()) + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + etcdLatest := &druidv1alpha1.Etcd{} + if err := l.client.Get(ctx, etcd.GetNamespaceName(), etcdLatest); err != nil { + return err + } - // update last operation - if etcd.Status.LastOperation == nil { - etcd.Status.LastOperation = &druidv1alpha1.LastOperation{} - } - etcd.Status.LastOperation.RunID = ctx.RunID - etcd.Status.LastOperation.Type = operationType - etcd.Status.LastOperation.State = operationState - etcd.Status.LastOperation.LastUpdateTime = metav1.NewTime(time.Now().UTC()) - etcd.Status.LastOperation.Description = description - // update last errors - etcd.Status.LastErrors = lastErrors + // Apply changes to the latest version of etcd + etcdLatest.Status.LastOperation = &druidv1alpha1.LastOperation{ + RunID: ctx.RunID, + Type: operationType, + State: operationState, + LastUpdateTime: metav1.NewTime(time.Now().UTC()), + Description: description, + } + etcdLatest.Status.LastErrors = lastErrors + + return l.client.Status().Update(ctx, etcdLatest) + }) - err := l.client.Status().Patch(ctx, etcd, etcdPatch) - if err != nil { - l.logger.Error(err, "failed to update LastOperation and LastErrors") + if retryErr != nil { + l.logger.Error(retryErr, "Failed to update etcd status.LastOperation after retries") } - return err + + return retryErr } From 7da0ce5f5e32c16da426c082c95e2a85a2971ce4 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 1 Jan 2024 10:44:08 +0530 Subject: [PATCH 034/235] WIP: Refactor Reconcile function --- internal/controller/etcd/reconcile_spec.go | 129 +++++++++++++++- internal/controller/etcd/reconciler.go | 166 +++++++++++---------- 2 files changed, 209 insertions(+), 86 deletions(-) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index c367f1e82..b8d2940e7 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -16,24 +16,139 @@ package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/controller/utils" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/operator" + "github.com/gardener/etcd-druid/internal/operator/resource" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) -func (r *Reconciler) triggerReconcileSpec() error { - return nil +// reconcileStepFn is a step in the reconcile spec flow. Every step must have this signature. +type reconcileStepFn func(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult + +func (r *Reconciler) triggerReconcileSpec(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { + reconcileStepFns := []reconcileStepFn{ + r.recordReconcileStartOperation, + r.syncEtcdResources, + r.removeOperationAnnotation, + r.recordReconcileSuccessOperation, + } + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + + for _, fn := range reconcileStepFns { + if stepResult := fn(ctx, logger, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { + return r.recordIncompleteReconcileOperation(ctx, logger, etcdObjectKey, stepResult) + } + } + return ctrlutils.DoNotRequeue() +} + +func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if _, ok := etcd.Annotations[v1beta1constants.GardenerOperation]; ok { + logger.Info("Removing operation annotation") + withOpAnnotation := etcd.DeepCopy() + delete(etcd.Annotations, v1beta1constants.GardenerOperation) + if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { + return utils.ReconcileWithError(err) + } + } + return ctrlutils.ContinueReconcile() +} +func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + resourceOperators := r.getOrderedOperatorsForSync() + for _, kind := range resourceOperators { + op := r.operatorRegistry.GetOperator(kind) + if err := op.Sync(ctx, etcd); err != nil { + return utils.ReconcileWithError(err) + } + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if err := r.lastOpErrRecorder.RecordStart(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile); err != nil { + logger.Error(err, "failed to record etcd reconcile start operation") + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if err := r.lastOpErrRecorder.RecordSuccess(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile); err != nil { + logger.Error(err, "failed to record etcd reconcile success operation") + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() } +func (r *Reconciler) recordIncompleteReconcileOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if err := r.lastOpErrRecorder.RecordError(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { + logger.Error(err, "failed to record last operation and last errors for etcd reconcilation") + return ctrlutils.ReconcileWithError(err) + } + return exitReconcileStepResult +} + +// canReconcileSpec assesses whether the Etcd spec should undergo reconciliation. +// +// Reconciliation decision follows these rules: +// - Skipped if 'druid.gardener.cloud/suspend-etcd-spec-reconcile' annotation is present, signaling a pause in reconciliation. +// - Also skipped if the deprecated 'druid.gardener.cloud/ignore-reconciliation' annotation is set. +// - Automatic reconciliation occurs if EnableEtcdSpecAutoReconcile is true. +// - If 'gardener.cloud/operation: reconcile' annotation exists and neither 'druid.gardener.cloud/suspend-etcd-spec-reconcile' nor the deprecated 'druid.gardener.cloud/ignore-reconciliation' is set to true, reconciliation proceeds upon Etcd spec changes. +// - Reconciliation is not initiated if EnableEtcdSpecAutoReconcile is false and none of the relevant annotations are present. func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { - //Check if spec reconciliation has been suspended, if yes, then record the event and return false. + // Check if spec reconciliation has been suspended, if yes, then record the event and return false. if suspendReconcileAnnotKey := r.getSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil { r.recordEtcdSpecReconcileSuspension(etcd, *suspendReconcileAnnotKey) return false } - // Check if - return true + + // Prefer using EnableEtcdSpecAutoReconcile for automatic reconciliation. + if r.config.EnableEtcdSpecAutoReconcile { + return true + } + + // Fallback to deprecated IgnoreOperationAnnotation if EnableEtcdSpecAutoReconcile is false. + if r.config.IgnoreOperationAnnotation { + return true + } + + // Reconcile if the 'reconcile-op' annotation is present. + if hasOperationAnnotationToReconcile(etcd) { + return true + } + + // Default case: Do not reconcile. + return false } // getSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent @@ -74,3 +189,7 @@ func (r *Reconciler) getOrderedOperatorsForSync() []operator.Kind { operator.StatefulSetKind, } } + +func hasOperationAnnotationToReconcile(etcd *druidv1alpha1.Etcd) bool { + return etcd.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile +} diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 5ceeb554f..ae63a4274 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -16,10 +16,12 @@ package etcd import ( "context" + "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/health/status" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/clientservice" "github.com/gardener/etcd-druid/internal/operator/configmap" @@ -32,12 +34,14 @@ import ( "github.com/gardener/etcd-druid/internal/operator/serviceaccount" "github.com/gardener/etcd-druid/internal/operator/snapshotlease" "github.com/gardener/etcd-druid/internal/operator/statefulset" + pkgutils "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -89,21 +93,7 @@ func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logge return reg } -/* - If deletionTimestamp set: - triggerDeletionFlow(); if err then requeue - Else: - If ignore-reconciliation is set to true: - skip reconcileSpec() - Else If IgnoreOperationAnnotation flag is true: - always reconcileSpec() - Else If IgnoreOperationAnnotation flag is false and reconcile-op annotation is present: - reconcileSpec() - if err in getting etcd, return with requeue - if err in deploying any of the components, then record pending requeue - reconcileStatus() - requeue after minimum of X seconds (EtcdStatusSyncPeriod) and previously recorded requeue request -*/ +type reconcileFn func(ctx context.Context, objectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult // TODO: where/how is this being used? // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch @@ -118,20 +108,41 @@ func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logge // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=events,verbs=create;get;list +// Reconcile manages the reconciliation of the Etcd resource to align it with its desired specifications. +// +// The reconciliation process involves the following steps: +// 1. Deletion Handling: If the Etcd resource has a deletionTimestamp, initiate the deletion workflow. On error, requeue the request. +// 2. Reconciliation Decision: Determine whether the Etcd spec should be reconciled based on annotations and flags. +// 3. Status Reconciliation: Always update the status of the Etcd resource to reflect its current state. +// 4. Scheduled Requeue: Requeue the reconciliation request after a defined period (EtcdStatusSyncPeriod) to maintain sync. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - type reconcileFn func(ctx context.Context, objectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult - reconcileFns := []reconcileFn{ - r.reconcileEtcdDeletion, - r.reconcileSpec, - r.reconcileStatus, + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() } + + var reconcileFns []reconcileFn + // If the etcd resource is marked for deletion. + if etcd.DeletionTimestamp != nil { + reconcileFns = append(reconcileFns, r.reconcileEtcdDeletion) + } else { + // Check if we can reconcile spec. + if r.canReconcileSpec(etcd) { + reconcileFns = append(reconcileFns, r.reconcileSpec) + } + } + + // Always reconcile status. + reconcileFns = append(reconcileFns, r.reconcileStatus) + + // Execute the reconcile functions. runID := uuid.New().String() for _, fn := range reconcileFns { if result := fn(ctx, req.NamespacedName, runID); ctrlutils.ShortCircuitReconcileFlow(result) { return result.ReconcileResult() } } - return ctrlutils.DoNotRequeue().ReconcileResult() + return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Reqeue").ReconcileResult() } func (r *Reconciler) reconcileEtcdDeletion(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { @@ -141,40 +152,72 @@ func (r *Reconciler) reconcileEtcdDeletion(ctx context.Context, etcdObjectKey cl } if etcd.IsMarkedForDeletion() { operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) - dLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "delete") + dLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "delete").WithValues("Reconciler", runID) return r.triggerDeletionFlow(operatorCtx, dLog, etcdObjectKey) } return ctrlutils.ContinueReconcile() } func (r *Reconciler) reconcileSpec(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { + operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) + rLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileSpec").WithValues("Reconciler", runID) + return r.triggerReconcileSpec(operatorCtx, rLog, etcdObjectKey) +} + +func (r *Reconciler) reconcileStatus(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) - resourceOperators := r.getOrderedOperatorsForSync() - for _, kind := range resourceOperators { - op := r.operatorRegistry.GetOperator(kind) - if err := op.Sync(operatorCtx, etcd); err != nil { - return utils.ReconcileWithError(err) + statusCheck := status.NewChecker(r.client, 5*time.Minute, 1*time.Minute) + if err := statusCheck.Check(ctx, r.logger, etcd); err != nil { + r.logger.Error(err, "Error executing status checks") + return utils.ReconcileWithError(err) + } + + r.logger.Info("Updating etcd status with statefulset information", "namespace", etcd.Namespace, "name", etcd.Name) + + sts, err := pkgutils.GetStatefulSet(ctx, r.client, etcd) + if err != nil { + return utils.ReconcileWithError(err) + } + + updateEtcdStatus := func() error { + latestetcd := etcd.DeepCopy() + if err := r.client.Get(ctx, etcd.GetNamespaceName(), latestetcd); err != nil { + return err + } + latestetcd.Status = etcd.Status + if sts != nil { + latestetcd.Status.Etcd = &druidv1alpha1.CrossVersionObjectReference{ + APIVersion: sts.APIVersion, + Kind: sts.Kind, + Name: sts.Name, + } + ready, _ := pkgutils.IsStatefulSetReady(etcd.Spec.Replicas, sts) + latestetcd.Status.CurrentReplicas = sts.Status.CurrentReplicas + latestetcd.Status.ReadyReplicas = sts.Status.ReadyReplicas + latestetcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas + latestetcd.Status.Replicas = sts.Status.CurrentReplicas + latestetcd.Status.Ready = &ready + } else { + latestetcd.Status.CurrentReplicas = 0 + latestetcd.Status.ReadyReplicas = 0 + latestetcd.Status.UpdatedReplicas = 0 + latestetcd.Status.Ready = pointer.Bool(false) } + + return r.client.Status().Update(ctx, latestetcd) } - return utils.ContinueReconcile() -} -func (r *Reconciler) reconcileStatus(ctx context.Context, etcdNamespacedName types.NamespacedName, runID string) ctrlutils.ReconcileStepResult { - /* - fetch EtcdMember resources - fetch member leases - status.condition checks - status.members checks - fetch latest Etcd resource - update etcd status - */ - - return ctrlutils.DoNotRequeue() + err = retry.RetryOnConflict(retry.DefaultRetry, updateEtcdStatus) + if err != nil { + r.logger.Error(err, "Failed to update etcd status") + return utils.ReconcileWithError(err) + } + + return utils.ContinueReconcile() } func (r *Reconciler) getLatestEtcd(ctx context.Context, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ctrlutils.ReconcileStepResult { @@ -186,42 +229,3 @@ func (r *Reconciler) getLatestEtcd(ctx context.Context, objectKey client.ObjectK } return ctrlutils.ContinueReconcile() } - -//func (r *Reconciler) runPreSpecReconcileChecks(etcd *druidv1alpha1.Etcd) ctrlutils.ReconcileStepResult { -// suspendEtcdSpecReconcileAnnotationKey := getSuspendEtcdReconcileAnnotationKey(etcd) -// if suspendEtcdSpecReconcileAnnotationKey != nil { -// r.recorder.Eventf( -// etcd, -// corev1.EventTypeWarning, -// "SpecReconciliationSkipped", -// "spec reconciliation of %s/%s is skipped by etcd-druid due to the presence of annotation %s on the etcd resource", -// etcd.Namespace, -// etcd.Name, -// suspendEtcdSpecReconcileAnnotationKey, -// ) -// -// } -//} - -//func (r *Reconciler) checkAndHandleReconcileSuspension(etcd *druidv1alpha1.Etcd) bool { -// //TODO: Once no one uses IgnoreReconciliationAnnotation annotation, then we can simplify this code. -// var annotationKey string -// if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) { -// annotationKey = druidv1alpha1.SuspendEtcdSpecReconcileAnnotation -// } else if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.IgnoreReconciliationAnnotation) { -// annotationKey = druidv1alpha1.IgnoreReconciliationAnnotation -// } -// if len(annotationKey) > 0 { -// r.recorder.Eventf( -// etcd, -// corev1.EventTypeWarning, -// "SpecReconciliationSkipped", -// "spec reconciliation of %s/%s is skipped by etcd-druid due to the presence of annotation %s on the etcd resource", -// etcd.Namespace, -// etcd.Name, -// annotationKey, -// ) -// return true -// } -// return ctrlutils.ContinueReconcile() -//} From 403e8ee59c104000aaf228ebaded234de622691c Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 1 Jan 2024 10:51:51 +0530 Subject: [PATCH 035/235] WIP: Refactor custodian controller --- internal/controller/custodian/reconciler.go | 26 ++------------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/internal/controller/custodian/reconciler.go b/internal/controller/custodian/reconciler.go index 04eca8230..310783774 100644 --- a/internal/controller/custodian/reconciler.go +++ b/internal/controller/custodian/reconciler.go @@ -16,13 +16,11 @@ package custodian import ( "context" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,27 +55,7 @@ func NewReconciler(mgr manager.Manager, config *Config) *Reconciler { // Reconcile reconciles the etcd. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - r.logger.Info("Custodian controller reconciliation started") - etcd := &druidv1alpha1.Etcd{} - if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { - if errors.IsNotFound(err) { - // Object not found, return. Created objects are automatically garbage collected. - // For additional cleanup logic use finalizers. - return ctrl.Result{}, nil - } - // Error reading the object - requeue the request. - return ctrl.Result{}, err - } - - logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) - - if !metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) && - etcd.Status.LastOperation != nil && etcd.Status.LastOperation.State != druidv1alpha1.LastOperationStateProcessing { - if err := r.triggerEtcdReconcile(ctx, logger, etcd); err != nil { - return ctrl.Result{Requeue: true}, err - } - } - + // TODO: @seshachalam-yv Already etcd-controller itself updating the etcd's status. return ctrl.Result{}, nil } From c01f31a6cfa4e19694b397312dd7024c08bf8aa7 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 1 Jan 2024 16:27:00 +0530 Subject: [PATCH 036/235] misc fixes --- internal/controller/etcd/reconcile_delete.go | 25 +++--- internal/controller/etcd/reconcile_spec.go | 36 ++++---- internal/controller/etcd/reconciler.go | 89 ++++++++++--------- .../operator/clientservice/clientservice.go | 7 +- .../clientservice/clientservice_test.go | 5 +- internal/operator/configmap/configmap.go | 7 +- internal/operator/memberlease/memberlease.go | 11 +-- internal/operator/peerservice/peerservice.go | 5 +- internal/operator/resource/types.go | 4 + 9 files changed, 87 insertions(+), 102 deletions(-) diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index a7da6c643..45ef07821 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -14,26 +14,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// deleteStepFn is a step in the deletion flow. Every deletion step must have this signature. -type deleteStepFn func(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult - // triggerDeletionFlow is the entry point for the deletion flow triggered for an etcd resource which has a DeletionTimeStamp set on it. func (r *Reconciler) triggerDeletionFlow(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { - deleteStepFns := []deleteStepFn{ + deleteStepFns := []reconcileFn{ r.recordDeletionStartOperation, r.deleteEtcdResources, r.verifyNoResourcesAwaitCleanUp, r.removeFinalizer, } for _, fn := range deleteStepFns { - if stepResult := fn(ctx, logger, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { + if stepResult := fn(ctx, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { return r.recordIncompleteDeletionOperation(ctx, logger, etcdObjectKey, stepResult) } } return ctrlutils.DoNotRequeue() } -func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -49,14 +46,14 @@ func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, logger lo }, }) } - logger.Info("triggering triggerDeletionFlow operators for all resources") + ctx.Logger.Info("triggering triggerDeletionFlow operators for all resources") if errs := utils.RunConcurrently(ctx, deleteTasks); len(errs) > 0 { return ctrlutils.ReconcileWithError(errs...) } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -71,32 +68,32 @@ func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx resource.OperatorContext, resourceNamesAwaitingCleanup = append(resourceNamesAwaitingCleanup, existingResourceNames...) } if len(resourceNamesAwaitingCleanup) > 0 { - logger.Info("Cleanup of all resources has not yet been completed", "resourceNamesAwaitingCleanup", resourceNamesAwaitingCleanup) + ctx.Logger.Info("Cleanup of all resources has not yet been completed", "resourceNamesAwaitingCleanup", resourceNamesAwaitingCleanup) return ctrlutils.ReconcileAfter(5*time.Second, "Cleanup of all resources has not yet been completed. Skipping removal of Finalizer") } - logger.Info("All resources have been cleaned up") + ctx.Logger.Info("All resources have been cleaned up") return ctrlutils.ContinueReconcile() } -func (r *Reconciler) removeFinalizer(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) removeFinalizer(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - logger.Info("Removing finalizer", "finalizerName", common.FinalizerName) + ctx.Logger.Info("Removing finalizer", "finalizerName", common.FinalizerName) if err := controllerutils.RemoveFinalizers(ctx, r.client, etcd, common.FinalizerName); client.IgnoreNotFound(err) != nil { return ctrlutils.ReconcileWithError(err) } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordDeletionStartOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordDeletionStartOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if err := r.lastOpErrRecorder.RecordStart(ctx, etcd, druidv1alpha1.LastOperationTypeDelete); err != nil { - logger.Error(err, "failed to record etcd deletion start operation") + ctx.Logger.Error(err, "failed to record etcd deletion start operation") return ctrlutils.ReconcileWithError(err) } return ctrlutils.ContinueReconcile() diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index b8d2940e7..6f94310e9 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -15,28 +15,26 @@ package etcd import ( + "time" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/resource" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) -// reconcileStepFn is a step in the reconcile spec flow. Every step must have this signature. -type reconcileStepFn func(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult - -func (r *Reconciler) triggerReconcileSpec(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { - reconcileStepFns := []reconcileStepFn{ +func (r *Reconciler) triggerReconcileSpecFlow(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { + reconcileStepFns := []reconcileFn{ r.recordReconcileStartOperation, r.syncEtcdResources, - r.removeOperationAnnotation, r.recordReconcileSuccessOperation, + r.removeOperationAnnotation, } etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { @@ -44,20 +42,20 @@ func (r *Reconciler) triggerReconcileSpec(ctx resource.OperatorContext, logger l } for _, fn := range reconcileStepFns { - if stepResult := fn(ctx, logger, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { - return r.recordIncompleteReconcileOperation(ctx, logger, etcdObjectKey, stepResult) + if stepResult := fn(ctx, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { + return r.recordIncompleteReconcileOperation(ctx, etcdObjectKey, stepResult) } } - return ctrlutils.DoNotRequeue() + return ctrlutils.ReconcileAfter(5*time.Second, "requeue with delay for status updates") } -func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if _, ok := etcd.Annotations[v1beta1constants.GardenerOperation]; ok { - logger.Info("Removing operation annotation") + ctx.Logger.Info("Removing operation annotation") withOpAnnotation := etcd.DeepCopy() delete(etcd.Annotations, v1beta1constants.GardenerOperation) if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { @@ -66,7 +64,7 @@ func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, log } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -81,37 +79,37 @@ func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, logger logr return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if err := r.lastOpErrRecorder.RecordStart(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile); err != nil { - logger.Error(err, "failed to record etcd reconcile start operation") + ctx.Logger.Error(err, "failed to record etcd reconcile start operation") return ctrlutils.ReconcileWithError(err) } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if err := r.lastOpErrRecorder.RecordSuccess(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile); err != nil { - logger.Error(err, "failed to record etcd reconcile success operation") + ctx.Logger.Error(err, "failed to record etcd reconcile success operation") return ctrlutils.ReconcileWithError(err) } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordIncompleteReconcileOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordIncompleteReconcileOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if err := r.lastOpErrRecorder.RecordError(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { - logger.Error(err, "failed to record last operation and last errors for etcd reconcilation") + ctx.Logger.Error(err, "failed to record last operation and last errors for etcd reconcilation") return ctrlutils.ReconcileWithError(err) } return exitReconcileStepResult diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index ae63a4274..703c84825 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -78,22 +78,7 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { }, nil } -func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logger, config *Config, imageVector imagevector.ImageVector) operator.Registry { - reg := operator.NewRegistry() - reg.Register(operator.ConfigMapKind, configmap.New(client, logger)) - reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount)) - reg.Register(operator.MemberLeaseKind, memberlease.New(client, logger)) - reg.Register(operator.SnapshotLeaseKind, snapshotlease.New(client, logger)) - reg.Register(operator.ClientServiceKind, clientservice.New(client, logger)) - reg.Register(operator.PeerServiceKind, peerservice.New(client, logger)) - reg.Register(operator.PodDisruptionBudgetKind, poddistruptionbudget.New(client, logger)) - reg.Register(operator.RoleKind, role.New(client, logger)) - reg.Register(operator.RoleBindingKind, rolebinding.New(client, logger)) - reg.Register(operator.StatefulSetKind, statefulset.New(client, logger, imageVector, config.FeatureGates)) - return reg -} - -type reconcileFn func(ctx context.Context, objectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult +type reconcileFn func(ctx resource.OperatorContext, objectKey client.ObjectKey) ctrlutils.ReconcileStepResult // TODO: where/how is this being used? // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch @@ -112,7 +97,7 @@ type reconcileFn func(ctx context.Context, objectKey client.ObjectKey, runID str // // The reconciliation process involves the following steps: // 1. Deletion Handling: If the Etcd resource has a deletionTimestamp, initiate the deletion workflow. On error, requeue the request. -// 2. Reconciliation Decision: Determine whether the Etcd spec should be reconciled based on annotations and flags. +// 2. Spec Reconciliation : Determine whether the Etcd spec should be reconciled based on annotations and flags and if there is a need then reconcile spec. // 3. Status Reconciliation: Always update the status of the Etcd resource to reflect its current state. // 4. Scheduled Requeue: Requeue the reconciliation request after a defined period (EtcdStatusSyncPeriod) to maintain sync. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -120,63 +105,79 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if result := r.getLatestEtcd(ctx, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result.ReconcileResult() } - - var reconcileFns []reconcileFn - // If the etcd resource is marked for deletion. - if etcd.DeletionTimestamp != nil { - reconcileFns = append(reconcileFns, r.reconcileEtcdDeletion) - } else { - // Check if we can reconcile spec. - if r.canReconcileSpec(etcd) { - reconcileFns = append(reconcileFns, r.reconcileSpec) - } + reconcileFns := []reconcileFn{ + r.reconcileEtcdDeletion, + r.reconcileSpec, + r.reconcileStatus, } - - // Always reconcile status. - reconcileFns = append(reconcileFns, r.reconcileStatus) - // Execute the reconcile functions. runID := uuid.New().String() + operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) for _, fn := range reconcileFns { - if result := fn(ctx, req.NamespacedName, runID); ctrlutils.ShortCircuitReconcileFlow(result) { + if result := fn(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { return result.ReconcileResult() } } - return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Reqeue").ReconcileResult() + + return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult() } -func (r *Reconciler) reconcileEtcdDeletion(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { +func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logger, config *Config, imageVector imagevector.ImageVector) operator.Registry { + reg := operator.NewRegistry() + reg.Register(operator.ConfigMapKind, configmap.New(client)) + reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount)) + reg.Register(operator.MemberLeaseKind, memberlease.New(client)) + reg.Register(operator.SnapshotLeaseKind, snapshotlease.New(client, logger)) + reg.Register(operator.ClientServiceKind, clientservice.New(client)) + reg.Register(operator.PeerServiceKind, peerservice.New(client)) + reg.Register(operator.PodDisruptionBudgetKind, poddistruptionbudget.New(client, logger)) + reg.Register(operator.RoleKind, role.New(client, logger)) + reg.Register(operator.RoleBindingKind, rolebinding.New(client, logger)) + reg.Register(operator.StatefulSetKind, statefulset.New(client, logger, imageVector, config.FeatureGates)) + return reg +} + +func (r *Reconciler) reconcileEtcdDeletion(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if etcd.IsMarkedForDeletion() { - operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) - dLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "delete").WithValues("Reconciler", runID) - return r.triggerDeletionFlow(operatorCtx, dLog, etcdObjectKey) + dLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "delete").WithValues("runId", ctx.RunID) + ctx.SetLogger(dLog) + return r.triggerDeletionFlow(ctx, dLog, etcdObjectKey) } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) reconcileSpec(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { - operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) - rLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileSpec").WithValues("Reconciler", runID) - return r.triggerReconcileSpec(operatorCtx, rLog, etcdObjectKey) +func (r *Reconciler) reconcileSpec(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if r.canReconcileSpec(etcd) { + rLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileSpec").WithValues("runID", ctx.RunID) + ctx.SetLogger(rLog) + return r.triggerReconcileSpecFlow(ctx, etcdObjectKey) + } + return ctrlutils.ContinueReconcile() } -func (r *Reconciler) reconcileStatus(ctx context.Context, etcdObjectKey client.ObjectKey, runID string) ctrlutils.ReconcileStepResult { +func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } + sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) + // TODO: Sesha to remove the hard coding for the timeout durations statusCheck := status.NewChecker(r.client, 5*time.Minute, 1*time.Minute) - if err := statusCheck.Check(ctx, r.logger, etcd); err != nil { + if err := statusCheck.Check(ctx, sLog, etcd); err != nil { r.logger.Error(err, "Error executing status checks") return utils.ReconcileWithError(err) } - r.logger.Info("Updating etcd status with statefulset information", "namespace", etcd.Namespace, "name", etcd.Name) + r.logger.Info("Updating etcd status with StatefulSet information", "namespace", etcd.Namespace, "name", etcd.Name) sts, err := pkgutils.GetStatefulSet(ctx, r.client, etcd) if err != nil { diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 4af37aac5..41d253253 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -6,7 +6,6 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,7 +27,6 @@ const ( type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -66,7 +64,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - r.logger.Info("Triggering delete of client service", "objectKey", objectKey) + ctx.Logger.Info("Triggering delete of client service") err := client.IgnoreNotFound(r.client.Delete(ctx, emptyClientService(objectKey))) return druiderr.WrapError( err, @@ -76,10 +74,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph ) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 0bc94483b..a3b2a14b3 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -32,7 +32,6 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log" ) // ------------------------ GetExistingResourceNames ------------------------ @@ -269,7 +268,7 @@ func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.With cl = fakeclient.NewClientBuilder().Build() } - op = New(cl, log.Log.WithName("test")) + op = New(cl) return } @@ -278,6 +277,6 @@ func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorCo ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") etcd = sampleEtcd() cl = mockclient.NewMockClient(gomock.NewController(t)) - op = New(cl, log.Log.WithName("test")) + op = New(cl) return } diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index e5fd6942b..776244d5c 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -7,7 +7,6 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" - "github.com/go-logr/logr" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -17,7 +16,6 @@ import ( type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -61,14 +59,13 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - r.logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) + ctx.Logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) return client.IgnoreNotFound(r.client.Delete(ctx, emptyConfigMap(objectKey))) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 9ab01ad83..dfc9f0424 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -1,8 +1,6 @@ package memberlease import ( - "context" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/operator/resource" @@ -11,7 +9,6 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,7 +18,6 @@ const purpose = "etcd-member-lease" type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -65,7 +61,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } -func (r _resource) doCreateOrUpdate(ctx context.Context, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { +func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { lease := emptyMemberLease(objKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { lease.Labels = utils.MergeMaps[string](utils.GetMemberLeaseLabels(etcd.Name), etcd.GetDefaultLabels()) @@ -75,7 +71,7 @@ func (r _resource) doCreateOrUpdate(ctx context.Context, etcd *druidv1alpha1.Etc if err != nil { return err } - r.logger.Info("triggered creation of member lease", "lease", objKey, "operationResult", opResult) + ctx.Logger.Info("triggered creation of member lease", "lease", objKey, "operationResult", opResult) return nil } @@ -86,10 +82,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph client.MatchingLabels(etcd.GetDefaultLabels())) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 1eb5727e6..7a727cbb5 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -5,7 +5,6 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,7 +16,6 @@ const defaultServerPort = 2380 type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -53,10 +51,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return client.IgnoreNotFound(r.client.Delete(ctx, emptyPeerService(getObjectKey(etcd)))) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/resource/types.go b/internal/operator/resource/types.go index 4038e83e1..bad88745d 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/resource/types.go @@ -33,6 +33,10 @@ func NewOperatorContext(ctx context.Context, logger logr.Logger, runID string) O } } +func (o *OperatorContext) SetLogger(logger logr.Logger) { + o.Logger = logger +} + // Operator manages one or more resources of a specific Kind which are provisioned for an etcd cluster. type Operator interface { // GetExistingResourceNames gets all resources that currently exist that this Operator manages. From 5881a66f29a9d270d6b93c1ae28311c739889536 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 2 Jan 2024 10:13:51 +0530 Subject: [PATCH 037/235] Optimize StatefulSet retrieval in GetStatefulSet instead of using list --- pkg/utils/statefulset.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/utils/statefulset.go b/pkg/utils/statefulset.go index b25406aa5..09c204c42 100644 --- a/pkg/utils/statefulset.go +++ b/pkg/utils/statefulset.go @@ -10,8 +10,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -38,16 +37,18 @@ func IsStatefulSetReady(etcdReplicas int32, statefulSet *appsv1.StatefulSet) (bo // GetStatefulSet fetches StatefulSet created for the etcd. func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { - statefulSets := &appsv1.StatefulSetList{} - if err := cl.List(ctx, statefulSets, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: labels.Set(etcd.GetDefaultLabels()).AsSelector()}); err != nil { - return nil, err - } + sts := &appsv1.StatefulSet{} - for _, sts := range statefulSets.Items { - if metav1.IsControlledBy(&sts, etcd) { - return &sts, nil + if err := cl.Get(ctx, client.ObjectKey{ + Name: etcd.Name, + Namespace: etcd.Namespace, + }, sts); err != nil { + + if apierrors.IsNotFound(err) { + return nil, nil } + return nil, err } - return nil, nil + return sts, nil } From 606409870a4ed1a89502a8f7bb09b1a42b92fd9b Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 2 Jan 2024 10:15:51 +0530 Subject: [PATCH 038/235] misc fixes --- internal/controller/etcd/reconcile_delete.go | 4 +- internal/controller/etcd/reconcile_spec.go | 6 +- internal/controller/etcd/reconciler.go | 53 +++++++---------- internal/controller/utils/etcdstatus.go | 59 ++++++++----------- .../operator/peerservice/peerservice_test.go | 5 +- internal/utils/image_test.go | 18 +++--- 6 files changed, 61 insertions(+), 84 deletions(-) diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 45ef07821..5e6eae694 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -92,7 +92,7 @@ func (r *Reconciler) recordDeletionStartOperation(ctx resource.OperatorContext, if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if err := r.lastOpErrRecorder.RecordStart(ctx, etcd, druidv1alpha1.LastOperationTypeDelete); err != nil { + if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeDelete); err != nil { ctx.Logger.Error(err, "failed to record etcd deletion start operation") return ctrlutils.ReconcileWithError(err) } @@ -104,7 +104,7 @@ func (r *Reconciler) recordIncompleteDeletionOperation(ctx resource.OperatorCont if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if err := r.lastOpErrRecorder.RecordError(ctx, etcd, druidv1alpha1.LastOperationTypeDelete, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { + if err := r.lastOpErrRecorder.RecordError(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeDelete, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { logger.Error(err, "failed to record last operation and last errors for etcd deletion") return ctrlutils.ReconcileWithError(err) } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 6f94310e9..399899d34 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -84,7 +84,7 @@ func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if err := r.lastOpErrRecorder.RecordStart(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile); err != nil { + if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile start operation") return ctrlutils.ReconcileWithError(err) } @@ -96,7 +96,7 @@ func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContex if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if err := r.lastOpErrRecorder.RecordSuccess(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile); err != nil { + if err := r.lastOpErrRecorder.RecordSuccess(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile success operation") return ctrlutils.ReconcileWithError(err) } @@ -108,7 +108,7 @@ func (r *Reconciler) recordIncompleteReconcileOperation(ctx resource.OperatorCon if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if err := r.lastOpErrRecorder.RecordError(ctx, etcd, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { + if err := r.lastOpErrRecorder.RecordError(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { ctx.Logger.Error(err, "failed to record last operation and last errors for etcd reconcilation") return ctrlutils.ReconcileWithError(err) } diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 703c84825..40437ebfe 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -16,6 +16,7 @@ package etcd import ( "context" + "log/slog" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -40,7 +41,6 @@ import ( "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/retry" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -170,51 +170,40 @@ func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey } sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) + slog.Info("Started etcd status update") // TODO: Sesha to remove the hard coding for the timeout durations statusCheck := status.NewChecker(r.client, 5*time.Minute, 1*time.Minute) if err := statusCheck.Check(ctx, sLog, etcd); err != nil { - r.logger.Error(err, "Error executing status checks") + sLog.Error(err, "Error executing status checks") return utils.ReconcileWithError(err) } - r.logger.Info("Updating etcd status with StatefulSet information", "namespace", etcd.Namespace, "name", etcd.Name) - sts, err := pkgutils.GetStatefulSet(ctx, r.client, etcd) if err != nil { return utils.ReconcileWithError(err) } - - updateEtcdStatus := func() error { - latestetcd := etcd.DeepCopy() - if err := r.client.Get(ctx, etcd.GetNamespaceName(), latestetcd); err != nil { - return err + if sts != nil { + etcd.Status.Etcd = &druidv1alpha1.CrossVersionObjectReference{ + APIVersion: sts.APIVersion, + Kind: sts.Kind, + Name: sts.Name, } - latestetcd.Status = etcd.Status - if sts != nil { - latestetcd.Status.Etcd = &druidv1alpha1.CrossVersionObjectReference{ - APIVersion: sts.APIVersion, - Kind: sts.Kind, - Name: sts.Name, - } - ready, _ := pkgutils.IsStatefulSetReady(etcd.Spec.Replicas, sts) - latestetcd.Status.CurrentReplicas = sts.Status.CurrentReplicas - latestetcd.Status.ReadyReplicas = sts.Status.ReadyReplicas - latestetcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas - latestetcd.Status.Replicas = sts.Status.CurrentReplicas - latestetcd.Status.Ready = &ready - } else { - latestetcd.Status.CurrentReplicas = 0 - latestetcd.Status.ReadyReplicas = 0 - latestetcd.Status.UpdatedReplicas = 0 - latestetcd.Status.Ready = pointer.Bool(false) - } - - return r.client.Status().Update(ctx, latestetcd) + ready, _ := pkgutils.IsStatefulSetReady(etcd.Spec.Replicas, sts) + etcd.Status.CurrentReplicas = sts.Status.CurrentReplicas + etcd.Status.ReadyReplicas = sts.Status.ReadyReplicas + etcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas + etcd.Status.Replicas = sts.Status.CurrentReplicas + etcd.Status.Ready = &ready + } else { + etcd.Status.CurrentReplicas = 0 + etcd.Status.ReadyReplicas = 0 + etcd.Status.UpdatedReplicas = 0 + etcd.Status.Ready = pointer.Bool(false) } - err = retry.RetryOnConflict(retry.DefaultRetry, updateEtcdStatus) + err = r.client.Status().Update(ctx, etcd) if err != nil { - r.logger.Error(err, "Failed to update etcd status") + sLog.Error(err, "Failed to update etcd status") return utils.ReconcileWithError(err) } diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index 5949a9f3a..b579323d0 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -8,15 +8,14 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) // LastOperationErrorRecorder records etcd.Status.LastOperation and etcd.Status.LastErrors type LastOperationErrorRecorder interface { - RecordStart(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error - RecordSuccess(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error - RecordError(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error + RecordStart(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error + RecordSuccess(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error + RecordError(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error } func NewLastOperationErrorRecorder(client client.Client, logger logr.Logger) LastOperationErrorRecorder { @@ -27,12 +26,11 @@ func NewLastOperationErrorRecorder(client client.Client, logger logr.Logger) Las } type lastOpErrRecorder struct { - client client.Client - objectKey client.ObjectKey - logger logr.Logger + client client.Client + logger logr.Logger } -func (l *lastOpErrRecorder) RecordStart(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error { +func (l *lastOpErrRecorder) RecordStart(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error { const ( etcdReconcileStarted string = "Etcd cluster reconciliation is in progress" etcdDeletionStarted string = "Etcd cluster deletion is in progress" @@ -44,10 +42,10 @@ func (l *lastOpErrRecorder) RecordStart(ctx resource.OperatorContext, etcd *drui case druidv1alpha1.LastOperationTypeDelete: description = etcdDeletionStarted } - return l.recordLastOperationAndErrors(ctx, etcd, operationType, druidv1alpha1.LastOperationStateProcessing, description) + return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateProcessing, description) } -func (l *lastOpErrRecorder) RecordSuccess(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType) error { +func (l *lastOpErrRecorder) RecordSuccess(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error { const ( etcdReconciledSuccessfully string = "Etcd cluster has been successfully reconciled" etcdDeletedSuccessfully string = "Etcd cluster has been successfully deleted" @@ -60,38 +58,29 @@ func (l *lastOpErrRecorder) RecordSuccess(ctx resource.OperatorContext, etcd *dr description = etcdDeletedSuccessfully } - return l.recordLastOperationAndErrors(ctx, etcd, operationType, druidv1alpha1.LastOperationStateSucceeded, description) + return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateSucceeded, description) } -func (l *lastOpErrRecorder) RecordError(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { +func (l *lastOpErrRecorder) RecordError(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { description += " Operation will be retried." lastErrors := druiderr.MapToLastErrors(errs) - return l.recordLastOperationAndErrors(ctx, etcd, operationType, druidv1alpha1.LastOperationStateError, description, lastErrors...) + return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateError, description, lastErrors...) } -func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, operationType druidv1alpha1.LastOperationType, operationState druidv1alpha1.LastOperationState, description string, lastErrors ...druidv1alpha1.LastError) error { - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - etcdLatest := &druidv1alpha1.Etcd{} - if err := l.client.Get(ctx, etcd.GetNamespaceName(), etcdLatest); err != nil { - return err - } - - // Apply changes to the latest version of etcd - etcdLatest.Status.LastOperation = &druidv1alpha1.LastOperation{ - RunID: ctx.RunID, - Type: operationType, - State: operationState, - LastUpdateTime: metav1.NewTime(time.Now().UTC()), - Description: description, - } - etcdLatest.Status.LastErrors = lastErrors - - return l.client.Status().Update(ctx, etcdLatest) - }) +func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, operationState druidv1alpha1.LastOperationState, description string, lastErrors ...druidv1alpha1.LastError) error { + etcd := &druidv1alpha1.Etcd{} + if err := l.client.Get(ctx, etcdObjectKey, etcd); err != nil { + return err + } - if retryErr != nil { - l.logger.Error(retryErr, "Failed to update etcd status.LastOperation after retries") + etcd.Status.LastOperation = &druidv1alpha1.LastOperation{ + RunID: ctx.RunID, + Type: operationType, + State: operationState, + LastUpdateTime: metav1.NewTime(time.Now().UTC()), + Description: description, } + etcd.Status.LastErrors = lastErrors - return retryErr + return l.client.Status().Update(ctx, etcd) } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 0af2f617b..189a650ce 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -32,7 +32,6 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log" ) // ------------------------ GetExistingResourceNames ------------------------ @@ -223,7 +222,7 @@ func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.With cl = fakeclient.NewClientBuilder().Build() } - op = New(cl, log.Log.WithName("test")) + op = New(cl) return } @@ -232,6 +231,6 @@ func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorCo ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") etcd = sampleEtcd() cl = mockclient.NewMockClient(gomock.NewController(t)) - op = New(cl, log.Log.WithName("test")) + op = New(cl) return } diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index fcc8cc79f..43aae31bc 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -48,7 +48,7 @@ var _ = Describe("Image retrieval tests", func() { Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) }) It("etcd spec has no image defined and image vector has both images set", func() { @@ -62,14 +62,14 @@ var _ = Describe("Image retrieval tests", func() { Expect(etcdImage).ToNot(BeNil()) vectorEtcdImage, err := imageVector.FindImage(common.Etcd) Expect(err).To(BeNil()) - Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) + Expect(etcdImage).To(Equal(vectorEtcdImage.String())) Expect(etcdBackupRestoreImage).ToNot(BeNil()) vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestore) Expect(err).To(BeNil()) - Expect(*etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) + Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) }) It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { @@ -82,12 +82,12 @@ var _ = Describe("Image retrieval tests", func() { Expect(etcdImage).ToNot(BeNil()) vectorEtcdImage, err := imageVector.FindImage(common.Etcd) Expect(err).To(BeNil()) - Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) + Expect(etcdImage).To(Equal(vectorEtcdImage.String())) Expect(etcdBackupRestoreImage).ToNot(BeNil()) Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) }) It("both spec and image vector do not have backup-restore image", func() { @@ -113,14 +113,14 @@ var _ = Describe("Image retrieval tests", func() { Expect(etcdImage).ToNot(BeNil()) vectorEtcdImage, err := imageVector.FindImage(common.EtcdWrapper) Expect(err).To(BeNil()) - Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) + Expect(etcdImage).To(Equal(vectorEtcdImage.String())) Expect(etcdBackupRestoreImage).ToNot(BeNil()) vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestoreDistroless) Expect(err).To(BeNil()) - Expect(*etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) + Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) + Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) }) }) From eaebc2f4cad86fcaba262ea3ad8854af9c119c85 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Tue, 2 Jan 2024 13:42:25 +0530 Subject: [PATCH 039/235] Remove custodian controller; status updates now handled by etcd controller itself --- internal/controller/config.go | 10 --- internal/controller/custodian/config.go | 44 -------------- internal/controller/custodian/reconciler.go | 67 --------------------- internal/controller/custodian/register.go | 61 ------------------- internal/controller/etcd/config.go | 19 ++++++ internal/controller/etcd/reconciler.go | 3 +- internal/controller/manager.go | 10 --- 7 files changed, 20 insertions(+), 194 deletions(-) delete mode 100644 internal/controller/custodian/config.go delete mode 100644 internal/controller/custodian/reconciler.go delete mode 100644 internal/controller/custodian/register.go diff --git a/internal/controller/config.go b/internal/controller/config.go index 0e165535e..7f9b41d06 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -18,7 +18,6 @@ import ( "fmt" "github.com/gardener/etcd-druid/internal/controller/compaction" - "github.com/gardener/etcd-druid/internal/controller/custodian" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" @@ -66,8 +65,6 @@ type ManagerConfig struct { FeatureGates featuregate.MutableFeatureGate // EtcdControllerConfig is the configuration required for etcd controller. EtcdControllerConfig *etcd.Config - // CustodianControllerConfig is the configuration required for custodian controller. - CustodianControllerConfig *custodian.Config // CompactionControllerConfig is the configuration required for compaction controller. CompactionControllerConfig *compaction.Config // EtcdCopyBackupsTaskControllerConfig is the configuration required for etcd-copy-backup-tasks controller. @@ -96,9 +93,6 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { cfg.EtcdControllerConfig = &etcd.Config{} etcd.InitFromFlags(fs, cfg.EtcdControllerConfig) - cfg.CustodianControllerConfig = &custodian.Config{} - custodian.InitFromFlags(fs, cfg.CustodianControllerConfig) - cfg.CompactionControllerConfig = &compaction.Config{} compaction.InitFromFlags(fs, cfg.CompactionControllerConfig) @@ -147,10 +141,6 @@ func (cfg *ManagerConfig) Validate() error { return err } - if err := cfg.CustodianControllerConfig.Validate(); err != nil { - return err - } - if err := cfg.CompactionControllerConfig.Validate(); err != nil { return err } diff --git a/internal/controller/custodian/config.go b/internal/controller/custodian/config.go deleted file mode 100644 index 0fa322326..000000000 --- a/internal/controller/custodian/config.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package custodian - -import ( - "github.com/gardener/etcd-druid/controllers/utils" - - flag "github.com/spf13/pflag" -) - -const ( - workersFlagName = "custodian-workers" - - defaultCustodianWorkers = 3 -) - -// Config contains configuration for the Custodian Controller. -type Config struct { - // Workers denotes the number of worker threads for the custodian controller. - Workers int -} - -// InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.IntVar(&cfg.Workers, workersFlagName, defaultCustodianWorkers, - "Number of worker threads for the custodian controller.") -} - -// Validate validates the config. -func (cfg *Config) Validate() error { - return utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers) -} diff --git a/internal/controller/custodian/reconciler.go b/internal/controller/custodian/reconciler.go deleted file mode 100644 index 310783774..000000000 --- a/internal/controller/custodian/reconciler.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package custodian - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -// Reconciler reconciles status of Etcd object -type Reconciler struct { - client.Client - scheme *runtime.Scheme - config *Config - logger logr.Logger -} - -// NewReconciler creates a new reconciler for Custodian. -func NewReconciler(mgr manager.Manager, config *Config) *Reconciler { - return &Reconciler{ - Client: mgr.GetClient(), - scheme: mgr.GetScheme(), - config: config, - logger: log.Log.WithName(controllerName), - } -} - -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;patch -// +kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;watch -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch -// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch -// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch - -// Reconcile reconciles the etcd. -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // TODO: @seshachalam-yv Already etcd-controller itself updating the etcd's status. - return ctrl.Result{}, nil -} - -func (r *Reconciler) triggerEtcdReconcile(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { - logger.Info("Adding operation annotation", "annotation", v1beta1constants.GardenerOperation) - withOpAnnotation := etcd.DeepCopy() - withOpAnnotation.Annotations[v1beta1constants.GardenerOperation] = v1beta1constants.GardenerOperationReconcile - return r.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)) -} diff --git a/internal/controller/custodian/register.go b/internal/controller/custodian/register.go deleted file mode 100644 index 31be80bb2..000000000 --- a/internal/controller/custodian/register.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package custodian - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/internal/controller/predicate" - appsv1 "k8s.io/api/apps/v1" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - rbacv1 "k8s.io/api/rbac/v1" - ctrl "sigs.k8s.io/controller-runtime" - ctrlbuilder "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -const controllerName = "custodian-controller" - -// RegisterWithManager registers the Custodian Controller with the given controller manager. -func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { - return ctrl. - NewControllerManagedBy(mgr). - Named(controllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.config.Workers, - }). - For(&druidv1alpha1.Etcd{}). - Owns(&corev1.Service{}). - Owns(&corev1.ConfigMap{}). - Owns(&coordinationv1.Lease{}). - Owns(&policyv1.PodDisruptionBudget{}). - Owns(&corev1.ServiceAccount{}). - Owns(&rbacv1.Role{}). - Owns(&rbacv1.RoleBinding{}). - Owns(&appsv1.StatefulSet{}, - // ignore (VPA) updates to statefulset container resources - ctrlbuilder.WithPredicates( - predicate.And( - druidpredicates.StatefulSetSpecChange(), - predicate.Not( - druidpredicates.StatefulSetContainerResourcesChange(), - ), - ), - ), - ). - Complete(r) -} diff --git a/internal/controller/etcd/config.go b/internal/controller/etcd/config.go index fb7b9fdbf..96eb9e3c9 100644 --- a/internal/controller/etcd/config.go +++ b/internal/controller/etcd/config.go @@ -17,6 +17,8 @@ const ( enableEtcdSpecAutoReconcileFlagName = "enable-etcd-spec-auto-reconcile" disableEtcdServiceAccountAutomountFlagName = "disable-etcd-serviceaccount-automount" etcdStatusSyncPeriodFlagName = "etcd-status-sync-period" + etcdMemberNotReadyThresholdFlagName = "etcd-member-notready-threshold" + etcdMemberUnknownThresholdFlagName = "etcd-member-unknown-threshold" ) const ( @@ -25,6 +27,8 @@ const ( defaultDisableEtcdServiceAccountAutomount = false defaultEnableEtcdSpecAutoReconcile = false defaultEtcdStatusSyncPeriod = 15 * time.Second + defaultEtcdMemberNotReadyThreshold = 5 * time.Minute + defaultEtcdMemberUnknownThreshold = 1 * time.Minute ) // featureList holds the feature gate names that are relevant for the Etcd Controller. @@ -53,6 +57,17 @@ type Config struct { EtcdStatusSyncPeriod time.Duration // FeatureGates contains the feature gates to be used by Etcd Controller. FeatureGates map[featuregate.Feature]bool + + // EtcdMember holds configuration related to etcd members. + EtcdMember EtcdMemberConfig +} + +// EtcdMemberConfig holds configuration related to etcd members. +type EtcdMemberConfig struct { + // NotReadyThreshold is the duration after which an etcd member's state is considered `NotReady`. + NotReadyThreshold time.Duration + // UnknownThreshold is the duration after which an etcd member's state is considered `Unknown`. + UnknownThreshold time.Duration } // InitFromFlags initializes the config from the provided CLI flag set. @@ -67,6 +82,10 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { "If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets.") fs.DurationVar(&cfg.EtcdStatusSyncPeriod, etcdStatusSyncPeriodFlagName, defaultEtcdStatusSyncPeriod, "Period after which an etcd status sync will be attempted.") + fs.DurationVar(&cfg.EtcdMember.NotReadyThreshold, etcdMemberNotReadyThresholdFlagName, defaultEtcdMemberNotReadyThreshold, + "Threshold after which an etcd member is considered not ready if the status was unknown before.") + fs.DurationVar(&cfg.EtcdMember.UnknownThreshold, etcdMemberUnknownThresholdFlagName, defaultEtcdMemberUnknownThreshold, + "Threshold after which an etcd member is considered unknown.") } // Validate validates the config. diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 40437ebfe..2a7e36730 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -17,7 +17,6 @@ package etcd import ( "context" "log/slog" - "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/controller/utils" @@ -172,7 +171,7 @@ func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) slog.Info("Started etcd status update") // TODO: Sesha to remove the hard coding for the timeout durations - statusCheck := status.NewChecker(r.client, 5*time.Minute, 1*time.Minute) + statusCheck := status.NewChecker(r.client, r.config.EtcdMember.NotReadyThreshold, r.config.EtcdMember.UnknownThreshold) if err := statusCheck.Check(ctx, sLog, etcd); err != nil { sLog.Error(err, "Error executing status checks") return utils.ReconcileWithError(err) diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 70a29d8eb..fe02afcd6 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -20,7 +20,6 @@ import ( "github.com/gardener/etcd-druid/internal/client/kubernetes" "github.com/gardener/etcd-druid/internal/controller/compaction" - "github.com/gardener/etcd-druid/internal/controller/custodian" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" @@ -91,15 +90,6 @@ func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) err return err } - // Add custodian reconciler to the manager - custodianReconciler := custodian.NewReconciler(mgr, config.CustodianControllerConfig) - if err != nil { - return err - } - if err = custodianReconciler.RegisterWithManager(mgr); err != nil { - return err - } - // Add compaction reconciler to the manager if the CLI flag enable-backup-compaction is true. if config.CompactionControllerConfig.EnableBackupCompaction { compactionReconciler, err := compaction.NewReconciler(mgr, config.CompactionControllerConfig) From f7f91e51953f878bfa9911cfce249d9cfb95a55f Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Wed, 3 Jan 2024 10:18:53 +0530 Subject: [PATCH 040/235] Replaced update operations with patch for enhanced efficiency and reduced conflicts --- internal/controller/etcd/reconcile_spec.go | 20 +------------------- internal/controller/utils/etcdstatus.go | 5 ++--- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 399899d34..da4e77e75 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -15,8 +15,6 @@ package etcd import ( - "time" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" @@ -36,17 +34,13 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx resource.OperatorContext, etcd r.recordReconcileSuccessOperation, r.removeOperationAnnotation, } - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result - } for _, fn := range reconcileStepFns { if stepResult := fn(ctx, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) { return r.recordIncompleteReconcileOperation(ctx, etcdObjectKey, stepResult) } } - return ctrlutils.ReconcileAfter(5*time.Second, "requeue with delay for status updates") + return ctrlutils.ContinueReconcile() } func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { @@ -80,10 +74,6 @@ func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, etcdObjKey } func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result - } if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile start operation") return ctrlutils.ReconcileWithError(err) @@ -92,10 +82,6 @@ func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, } func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result - } if err := r.lastOpErrRecorder.RecordSuccess(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile success operation") return ctrlutils.ReconcileWithError(err) @@ -104,10 +90,6 @@ func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContex } func (r *Reconciler) recordIncompleteReconcileOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result - } if err := r.lastOpErrRecorder.RecordError(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { ctx.Logger.Error(err, "failed to record last operation and last errors for etcd reconcilation") return ctrlutils.ReconcileWithError(err) diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index b579323d0..db159e0bf 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -72,7 +72,7 @@ func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorCo if err := l.client.Get(ctx, etcdObjectKey, etcd); err != nil { return err } - + originalEtcd := etcd.DeepCopy() etcd.Status.LastOperation = &druidv1alpha1.LastOperation{ RunID: ctx.RunID, Type: operationType, @@ -81,6 +81,5 @@ func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorCo Description: description, } etcd.Status.LastErrors = lastErrors - - return l.client.Status().Update(ctx, etcd) + return l.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)) } From 71fabca4c51bf3f46a19ed73be206076adfce635 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Wed, 3 Jan 2024 10:23:05 +0530 Subject: [PATCH 041/235] Added etcdReconcilePredicate for enhanced event filtering in etcd controller registration. --- internal/controller/etcd/reconciler.go | 9 ++-- internal/controller/etcd/register.go | 59 +++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 2a7e36730..ac3f8d6fe 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -16,7 +16,6 @@ package etcd import ( "context" - "log/slog" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/controller/utils" @@ -167,10 +166,10 @@ func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - + originalEtcd := etcd.DeepCopy() sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) - slog.Info("Started etcd status update") - // TODO: Sesha to remove the hard coding for the timeout durations + sLog.Info("Started etcd status update") + statusCheck := status.NewChecker(r.client, r.config.EtcdMember.NotReadyThreshold, r.config.EtcdMember.UnknownThreshold) if err := statusCheck.Check(ctx, sLog, etcd); err != nil { sLog.Error(err, "Error executing status checks") @@ -200,7 +199,7 @@ func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey etcd.Status.Ready = pointer.Bool(false) } - err = r.client.Status().Update(ctx, etcd) + err = r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)) if err != nil { sLog.Error(err, "Failed to update etcd status") return utils.ReconcileWithError(err) diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index ddfbb97a5..4d675bf20 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -4,9 +4,13 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) const controllerName = "etcd-controller" @@ -20,7 +24,60 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { MaxConcurrentReconciles: r.config.Workers, RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(10*time.Millisecond, r.config.EtcdStatusSyncPeriod), }). - For(&druidv1alpha1.Etcd{}) + For(&druidv1alpha1.Etcd{}). + WithEventFilter( + predicate.Or(r.etcdReconcilePredicate()), + ) return builder.Complete(r) } + +func (r *Reconciler) etcdReconcilePredicate() predicate.Predicate { + // shouldReconcileBasedOnReconcileAnnotation checks if the object should be reconciled based on its annotations. + shouldReconcileBasedOnReconcileAnnotation := func(obj client.Object) bool { + if r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation { + return true // Auto reconcile is enabled or operation annotations are ignored. + } + // Reconcile only if the GardenerOperation annotation is set to 'Reconcile'. + return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile + } + + // updatePredicate defines the logic to decide if an update event should trigger a reconcile. + updatePredicate := func(event event.UpdateEvent) bool { + if !shouldReconcileBasedOnReconcileAnnotation(event.ObjectNew) { + return false // Skip if the new object does not have the required annotation. + } + + // Reconcile if the old object did not have the reconcile annotation. + if !shouldReconcileBasedOnReconcileAnnotation(event.ObjectOld) { + return true // New reconcile annotation added to new object, hence reconcile required. + } + + // Detect if there's a change in the spec. + specChanged := event.ObjectNew.GetGeneration() != event.ObjectOld.GetGeneration() + if specChanged { + return true // Reconcile due to spec change. + } + + etcd, ok := event.ObjectNew.(*druidv1alpha1.Etcd) + if !ok { + return false + } + + if etcd.Status.LastOperation.Type == druidv1alpha1.LastOperationTypeReconcile && etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateSucceeded { + return false // Skip reconcile if the last reconcile operation was successful. + } + + // Calculate the time elapsed since the last update. + elapsedDuration := time.Since(etcd.Status.LastOperation.LastUpdateTime.UTC()) + return elapsedDuration > r.config.EtcdStatusSyncPeriod // Reconcile if the time elapsed is more than sync period. + } + + // Constructing the predicate.Funcs with specific functions for each event type. + return predicate.Funcs{ + UpdateFunc: updatePredicate, + CreateFunc: func(event event.CreateEvent) bool { return true }, + GenericFunc: func(event event.GenericEvent) bool { return shouldReconcileBasedOnReconcileAnnotation(event.Object) }, + DeleteFunc: func(event event.DeleteEvent) bool { return true }, + } +} From 54d8621951d6ccb1e18d4c81b77eeda4963f485d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 3 Jan 2024 12:44:20 +0530 Subject: [PATCH 042/235] added code to set ObservedGeneration and refactored predicates for etcd reconciler --- Makefile | 2 +- internal/controller/etcd/reconcile_spec.go | 15 +++ internal/controller/etcd/register.go | 91 +++++++++++-------- .../operator/clientservice/clientservice.go | 8 +- .../clientservice/clientservice_test.go | 8 ++ test/sample/etcd.go | 3 + 6 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 test/sample/etcd.go diff --git a/Makefile b/Makefile index 804619faa..c14b935d7 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ druid: fmt check # Run against the configured Kubernetes cluster in ~/.kube/config .PHONY: run -run: fmt check +run: go run ./main.go # Install CRDs into a cluster diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index da4e77e75..91e79a780 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -31,6 +31,7 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx resource.OperatorContext, etcd reconcileStepFns := []reconcileFn{ r.recordReconcileStartOperation, r.syncEtcdResources, + r.updateObservedGeneration, r.recordReconcileSuccessOperation, r.removeOperationAnnotation, } @@ -73,6 +74,20 @@ func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, etcdObjKey return ctrlutils.ContinueReconcile() } +func (r *Reconciler) updateObservedGeneration(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + originalEtcd := etcd.DeepCopy() + etcd.Status.ObservedGeneration = &etcd.Generation + if err := r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)); err != nil { + return ctrlutils.ReconcileWithError(err) + } + ctx.Logger.Info("patched status.ObservedGeneration", "ObservedGeneration", etcd.Generation) + return ctrlutils.ContinueReconcile() +} + func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile start operation") diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 4d675bf20..35281e747 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -5,6 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,58 +27,70 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { }). For(&druidv1alpha1.Etcd{}). WithEventFilter( - predicate.Or(r.etcdReconcilePredicate()), + predicate.Or( + predicate.And(r.forcedReconcile(), noSpecAndStatusUpdated()), + predicate.And(r.reconcilePermitted(), onlySpecUpdated()), + ), ) return builder.Complete(r) } -func (r *Reconciler) etcdReconcilePredicate() predicate.Predicate { - // shouldReconcileBasedOnReconcileAnnotation checks if the object should be reconciled based on its annotations. - shouldReconcileBasedOnReconcileAnnotation := func(obj client.Object) bool { - if r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation { - return true // Auto reconcile is enabled or operation annotations are ignored. - } - // Reconcile only if the GardenerOperation annotation is set to 'Reconcile'. - return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile +func onlySpecUpdated() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return hasSpecChanged(updateEvent) && !hasStatusChanged(updateEvent) + }, } +} - // updatePredicate defines the logic to decide if an update event should trigger a reconcile. - updatePredicate := func(event event.UpdateEvent) bool { - if !shouldReconcileBasedOnReconcileAnnotation(event.ObjectNew) { - return false // Skip if the new object does not have the required annotation. - } - - // Reconcile if the old object did not have the reconcile annotation. - if !shouldReconcileBasedOnReconcileAnnotation(event.ObjectOld) { - return true // New reconcile annotation added to new object, hence reconcile required. - } +func noSpecAndStatusUpdated() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return !hasSpecChanged(updateEvent) && !hasStatusChanged(updateEvent) + }, + } +} - // Detect if there's a change in the spec. - specChanged := event.ObjectNew.GetGeneration() != event.ObjectOld.GetGeneration() - if specChanged { - return true // Reconcile due to spec change. - } +func hasSpecChanged(updateEvent event.UpdateEvent) bool { + return updateEvent.ObjectNew.GetGeneration() != updateEvent.ObjectOld.GetGeneration() +} - etcd, ok := event.ObjectNew.(*druidv1alpha1.Etcd) - if !ok { - return false - } +func hasStatusChanged(updateEvent event.UpdateEvent) bool { + return apiequality.Semantic.DeepEqual(accessEtcdStatus(updateEvent.ObjectNew), accessEtcdStatus(updateEvent.ObjectOld)) +} - if etcd.Status.LastOperation.Type == druidv1alpha1.LastOperationTypeReconcile && etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateSucceeded { - return false // Skip reconcile if the last reconcile operation was successful. - } +func accessEtcdStatus(object client.Object) *druidv1alpha1.EtcdStatus { + if etcd, ok := object.(*druidv1alpha1.Etcd); !ok { + return nil + } else { + return &etcd.Status + } +} - // Calculate the time elapsed since the last update. - elapsedDuration := time.Since(etcd.Status.LastOperation.LastUpdateTime.UTC()) - return elapsedDuration > r.config.EtcdStatusSyncPeriod // Reconcile if the time elapsed is more than sync period. +// reconcilePermitted +func (r *Reconciler) reconcilePermitted() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + if r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation { + return true + } + return !hasReconcileAnnotation(updateEvent.ObjectOld) && hasReconcileAnnotation(updateEvent.ObjectNew) + }, } +} - // Constructing the predicate.Funcs with specific functions for each event type. +func (r *Reconciler) forcedReconcile() predicate.Predicate { return predicate.Funcs{ - UpdateFunc: updatePredicate, - CreateFunc: func(event event.CreateEvent) bool { return true }, - GenericFunc: func(event event.GenericEvent) bool { return shouldReconcileBasedOnReconcileAnnotation(event.Object) }, - DeleteFunc: func(event event.DeleteEvent) bool { return true }, + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + if r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation { + return false + } + return !hasReconcileAnnotation(updateEvent.ObjectOld) && hasReconcileAnnotation(updateEvent.ObjectNew) + }, } } + +func hasReconcileAnnotation(object client.Object) bool { + return object.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile +} diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 41d253253..461b4f311 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -44,7 +44,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { svc := emptyClientService(getObjectKey(etcd)) - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { + result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { svc.Labels = getLabels(etcd) svc.Annotations = getAnnotations(etcd) svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} @@ -55,6 +55,9 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) return nil }) + if err == nil { + ctx.Logger.Info("synced", "resource", "client-service", "name", svc.Name, "result", result) + } return druiderr.WrapError(err, ErrSyncingClientService, "Sync", @@ -66,6 +69,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of client service") err := client.IgnoreNotFound(r.client.Delete(ctx, emptyClientService(objectKey))) + if err == nil { + ctx.Logger.Info("deleted", "resource", "client-service", "name", objectKey.Name) + } return druiderr.WrapError( err, ErrDeletingClientService, diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index a3b2a14b3..7f186f10f 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -36,6 +36,14 @@ import ( // ------------------------ GetExistingResourceNames ------------------------ +//func TestGetExistingResourceNames(t *testing.T) { +// testcases := []struct { +// description string +// svcExists bool +// +// } +//} + func TestClientServiceGetExistingResourceNames_WithExistingService(t *testing.T) { g, ctx, etcd, _, op := setupWithFakeClient(t, true) diff --git a/test/sample/etcd.go b/test/sample/etcd.go new file mode 100644 index 000000000..28f85fcd9 --- /dev/null +++ b/test/sample/etcd.go @@ -0,0 +1,3 @@ +package sample + +// Add the etcd builder code from test/utils/etcd.go here From 575f0bba40bf54d131f6d3a4311cf4caa6c19093 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 3 Jan 2024 15:51:51 +0530 Subject: [PATCH 043/235] added sample package, refactored test for client-svc operator, introduced fakeclientbuilder --- .../clientservice/clientservice_test.go | 558 ++++++++++-------- test/sample/service.go | 58 ++ test/utils/fakeclient.go | 61 ++ 3 files changed, 423 insertions(+), 254 deletions(-) create mode 100644 test/sample/service.go create mode 100644 test/utils/fakeclient.go diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 7f186f10f..233e8d97b 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -16,275 +16,325 @@ package clientservice import ( "context" - "fmt" + "errors" "testing" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/operator/resource" - "github.com/gardener/etcd-druid/internal/utils" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" - "github.com/golang/mock/gomock" - "github.com/onsi/gomega" + "github.com/google/uuid" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + apierrors "k8s.io/apimachinery/pkg/api/errors" ) // ------------------------ GetExistingResourceNames ------------------------ -//func TestGetExistingResourceNames(t *testing.T) { -// testcases := []struct { -// description string -// svcExists bool -// -// } -//} - -func TestClientServiceGetExistingResourceNames_WithExistingService(t *testing.T) { - g, ctx, etcd, _, op := setupWithFakeClient(t, true) - - err := op.Sync(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - names, err := op.GetExistingResourceNames(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(names).To(gomega.ContainElement(etcd.GetClientServiceName())) -} - -func TestClientServiceGetExistingResourceNames_ServiceNotFound(t *testing.T) { - g, ctx, etcd, _, op := setupWithFakeClient(t, false) - - names, err := op.GetExistingResourceNames(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(names).To(gomega.BeEmpty()) -} - -func TestClientServiceGetExistingResourceNames_WithError(t *testing.T) { - g, ctx, etcd, cl, op := setupWithMockClient(t) - cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() - - names, err := op.GetExistingResourceNames(ctx, etcd) - g.Expect(err).To(gomega.HaveOccurred()) - g.Expect(names).To(gomega.BeEmpty()) -} - -// ----------------------------------- Sync ----------------------------------- - -func TestClientServiceSync_CreateNewService(t *testing.T) { - g, ctx, etcd, cl, op := setupWithFakeClient(t, false) - err := op.Sync(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetClientServiceName(), - Namespace: etcd.Namespace, - }, - } - err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) - g.Expect(err).NotTo(gomega.HaveOccurred()) - checkClientService(g, service, etcd) -} - -func TestClientServiceSync_UpdateExistingService(t *testing.T) { - g, ctx, etcd, cl, op := setupWithFakeClient(t, true) - etcd.Spec.Etcd.ServerPort = nil - etcd.Spec.Etcd.ClientPort = nil - etcd.Spec.Backup.Port = nil - - etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{ - Labels: map[string]string{"testingKey": "testingValue"}, - Annotations: map[string]string{"testingAnnotationKey": "testingValue"}, - } - - err := op.Sync(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetClientServiceName(), - Namespace: etcd.Namespace, - }, - } - err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(service).ToNot(gomega.BeNil()) - checkClientService(g, service, etcd) -} - -func TestClientServiceSync_WithClientError(t *testing.T) { - g, ctx, etcd, cl, op := setupWithMockClient(t) - cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() - cl.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake create error")).AnyTimes() - cl.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake update error")).AnyTimes() - - err := op.Sync(ctx, etcd) - g.Expect(err).To(gomega.HaveOccurred()) - g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrSyncingClientService))) -} - -// ----------------------------- TriggerDelete ------------------------------- - -func TestClientServiceTriggerDelete_ExistingService(t *testing.T) { - g, ctx, etcd, cl, op := setupWithFakeClient(t, true) - err := op.TriggerDelete(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - serviceList := &corev1.List{} - err = cl.List(ctx, serviceList) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(serviceList.Items).To(gomega.BeEmpty()) -} - -func TestClientServiceTriggerDelete_ServiceNotFound(t *testing.T) { - g, ctx, etcd, _, op := setupWithFakeClient(t, false) - - err := op.TriggerDelete(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) -} - -func TestClientServiceTriggerDelete_WithClientError(t *testing.T) { - g, ctx, etcd, cl, op := setupWithMockClient(t) - // Configure the mock client to return an error on delete operation - cl.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake delete error")).AnyTimes() - - err := op.TriggerDelete(ctx, etcd) - g.Expect(err).To(gomega.HaveOccurred()) - g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrDeletingClientService))) -} - -// ---------------------------- Helper Functions ----------------------------- - -func sampleEtcd() *druidv1alpha1.Etcd { - return &druidv1alpha1.Etcd{ - - ObjectMeta: metav1.ObjectMeta{ - Name: "test-etcd", - Namespace: "test-namespace", - UID: "xxx-yyy-zzz", - }, - Spec: druidv1alpha1.EtcdSpec{ +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) - Backup: druidv1alpha1.BackupSpec{ - Port: pointer.Int32(1111), - }, - Etcd: druidv1alpha1.EtcdConfig{ - ClientPort: pointer.Int32(2222), - ServerPort: pointer.Int32(3333), - }, +func TestGetExistingResourceNames(t *testing.T) { + internalErr := errors.New("test internal error") + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + t.Parallel() + testcases := []struct { + name string + svcExists bool + getErr *apierrors.StatusError + expectedErr error + expectedServiceNames []string + }{ + { + "should return the existing service name", + true, + nil, + nil, + []string{etcd.GetClientServiceName()}, }, - } -} - -func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) - - expectedLabels := etcd.GetDefaultLabels() - var expectedAnnotations map[string]string - if etcd.Spec.Etcd.ClientService != nil { - expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations - expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) - } - g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(svc.Annotations).To(gomega.Equal(expectedAnnotations)) - g.Expect(svc.Labels).To(gomega.Equal(expectedLabels)) - g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) - g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) - g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( - gomega.Equal(corev1.ServicePort{ - Name: "client", - Protocol: corev1.ProtocolTCP, - Port: clientPort, - TargetPort: intstr.FromInt(int(clientPort)), - }), - gomega.Equal(corev1.ServicePort{ - Name: "server", - Protocol: corev1.ProtocolTCP, - Port: peerPort, - TargetPort: intstr.FromInt(int(peerPort)), - }), - gomega.Equal(corev1.ServicePort{ - Name: "backuprestore", - Protocol: corev1.ProtocolTCP, - Port: backupPort, - TargetPort: intstr.FromInt(int(backupPort)), - }), - )) -} - -func existingClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { - var annotations, labels map[string]string - if etcd.Spec.Etcd.ClientService != nil { - annotations = etcd.Spec.Etcd.ClientService.Annotations - labels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) - - } - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetClientServiceName(), - Namespace: etcd.Namespace, - Annotations: annotations, - Labels: labels, - OwnerReferences: []metav1.OwnerReference{ - etcd.GetAsOwnerReference(), - }, + { + "should return empty slice when service is not found", + false, + apierrors.NewNotFound(corev1.Resource("services"), ""), + nil, + []string{}, }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, - SessionAffinity: corev1.ServiceAffinityNone, - Selector: etcd.GetDefaultLabels(), - PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "client", - Protocol: corev1.ProtocolTCP, - Port: *etcd.Spec.Etcd.ClientPort, - TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ClientPort)), - }, - { - Name: "server", - Protocol: corev1.ProtocolTCP, - Port: *etcd.Spec.Etcd.ServerPort, - TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), - }, - { - Name: "backuprestore", - Protocol: corev1.ProtocolTCP, - Port: *etcd.Spec.Backup.Port, - TargetPort: intstr.FromInt(int(*etcd.Spec.Backup.Port)), - }, - }, + { + "should return error when get fails", + true, + apierrors.NewInternalError(internalErr), + apierrors.NewInternalError(internalErr), + nil, }, } -} - -func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl client.Client, op resource.Operator) { - g = gomega.NewWithT(t) - ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") - etcd = sampleEtcd() - if withExistingService { - cl = fakeclient.NewClientBuilder().WithObjects(existingClientService(etcd)).Build() - } else { - cl = fakeclient.NewClientBuilder().Build() + g := NewWithT(t) + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.svcExists { + fakeClientBuilder.WithObjects(sample.NewClientService(etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + } else { + g.Expect(err).To(BeNil()) + } + g.Expect(svcNames, tc.expectedServiceNames) + }) } - op = New(cl) - return } -func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl *mockclient.MockClient, op resource.Operator) { - g = gomega.NewWithT(t) - ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") - etcd = sampleEtcd() - cl = mockclient.NewMockClient(gomock.NewController(t)) - op = New(cl) - return -} +//func TestClientServiceGetExistingResourceNames_WithExistingService(t *testing.T) { +// g, ctx, etcd, _, op := setupWithFakeClient(t, true) +// +// err := op.Sync(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// +// names, err := op.GetExistingResourceNames(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// g.Expect(names).To(gomega.ContainElement(etcd.GetClientServiceName())) +//} +// +//func TestClientServiceGetExistingResourceNames_ServiceNotFound(t *testing.T) { +// g, ctx, etcd, _, op := setupWithFakeClient(t, false) +// +// names, err := op.GetExistingResourceNames(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// g.Expect(names).To(gomega.BeEmpty()) +//} +// +//func TestClientServiceGetExistingResourceNames_WithError(t *testing.T) { +// g, ctx, etcd, cl, op := setupWithMockClient(t) +// cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() +// +// names, err := op.GetExistingResourceNames(ctx, etcd) +// g.Expect(err).To(gomega.HaveOccurred()) +// g.Expect(names).To(gomega.BeEmpty()) +//} +// +//// ----------------------------------- Sync ----------------------------------- +// +//func TestClientServiceSync_CreateNewService(t *testing.T) { +// g, ctx, etcd, cl, op := setupWithFakeClient(t, false) +// err := op.Sync(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// +// service := &corev1.Service{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: etcd.GetClientServiceName(), +// Namespace: etcd.Namespace, +// }, +// } +// err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// checkClientService(g, service, etcd) +//} +// +//func TestClientServiceSync_UpdateExistingService(t *testing.T) { +// g, ctx, etcd, cl, op := setupWithFakeClient(t, true) +// etcd.Spec.Etcd.ServerPort = nil +// etcd.Spec.Etcd.ClientPort = nil +// etcd.Spec.Backup.Port = nil +// +// etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{ +// Labels: map[string]string{"testingKey": "testingValue"}, +// Annotations: map[string]string{"testingAnnotationKey": "testingValue"}, +// } +// +// err := op.Sync(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// +// service := &corev1.Service{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: etcd.GetClientServiceName(), +// Namespace: etcd.Namespace, +// }, +// } +// err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// g.Expect(service).ToNot(gomega.BeNil()) +// checkClientService(g, service, etcd) +//} +// +//func TestClientServiceSync_WithClientError(t *testing.T) { +// g, ctx, etcd, cl, op := setupWithMockClient(t) +// cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() +// cl.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake create error")).AnyTimes() +// cl.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake update error")).AnyTimes() +// +// err := op.Sync(ctx, etcd) +// g.Expect(err).To(gomega.HaveOccurred()) +// g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrSyncingClientService))) +//} +// +//// ----------------------------- TriggerDelete ------------------------------- +// +//func TestClientServiceTriggerDelete_ExistingService(t *testing.T) { +// g, ctx, etcd, cl, op := setupWithFakeClient(t, true) +// err := op.TriggerDelete(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// +// serviceList := &corev1.List{} +// err = cl.List(ctx, serviceList) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +// g.Expect(serviceList.Items).To(gomega.BeEmpty()) +//} +// +//func TestClientServiceTriggerDelete_ServiceNotFound(t *testing.T) { +// g, ctx, etcd, _, op := setupWithFakeClient(t, false) +// +// err := op.TriggerDelete(ctx, etcd) +// g.Expect(err).NotTo(gomega.HaveOccurred()) +//} +// +//func TestClientServiceTriggerDelete_WithClientError(t *testing.T) { +// g, ctx, etcd, cl, op := setupWithMockClient(t) +// // Configure the mock client to return an error on delete operation +// cl.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake delete error")).AnyTimes() +// +// err := op.TriggerDelete(ctx, etcd) +// g.Expect(err).To(gomega.HaveOccurred()) +// g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrDeletingClientService))) +//} +// +//// ---------------------------- Helper Functions ----------------------------- +// +//func sampleEtcd() *druidv1alpha1.Etcd { +// return &druidv1alpha1.Etcd{ +// +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-etcd", +// Namespace: "test-namespace", +// UID: "xxx-yyy-zzz", +// }, +// Spec: druidv1alpha1.EtcdSpec{ +// +// Backup: druidv1alpha1.BackupSpec{ +// Port: pointer.Int32(1111), +// }, +// Etcd: druidv1alpha1.EtcdConfig{ +// ClientPort: pointer.Int32(2222), +// ServerPort: pointer.Int32(3333), +// }, +// }, +// } +//} +// +//func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { +// clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) +// backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) +// peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) +// +// expectedLabels := etcd.GetDefaultLabels() +// var expectedAnnotations map[string]string +// if etcd.Spec.Etcd.ClientService != nil { +// expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations +// expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) +// } +// g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) +// g.Expect(svc.Annotations).To(gomega.Equal(expectedAnnotations)) +// g.Expect(svc.Labels).To(gomega.Equal(expectedLabels)) +// g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) +// g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) +// g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) +// g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( +// gomega.Equal(corev1.ServicePort{ +// Name: "client", +// Protocol: corev1.ProtocolTCP, +// Port: clientPort, +// TargetPort: intstr.FromInt(int(clientPort)), +// }), +// gomega.Equal(corev1.ServicePort{ +// Name: "server", +// Protocol: corev1.ProtocolTCP, +// Port: peerPort, +// TargetPort: intstr.FromInt(int(peerPort)), +// }), +// gomega.Equal(corev1.ServicePort{ +// Name: "backuprestore", +// Protocol: corev1.ProtocolTCP, +// Port: backupPort, +// TargetPort: intstr.FromInt(int(backupPort)), +// }), +// )) +//} +// +//func existingClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { +// var annotations, labels map[string]string +// if etcd.Spec.Etcd.ClientService != nil { +// annotations = etcd.Spec.Etcd.ClientService.Annotations +// labels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) +// +// } +// return &corev1.Service{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: etcd.GetClientServiceName(), +// Namespace: etcd.Namespace, +// Annotations: annotations, +// Labels: labels, +// OwnerReferences: []metav1.OwnerReference{ +// etcd.GetAsOwnerReference(), +// }, +// }, +// Spec: corev1.ServiceSpec{ +// Type: corev1.ServiceTypeClusterIP, +// ClusterIP: corev1.ClusterIPNone, +// SessionAffinity: corev1.ServiceAffinityNone, +// Selector: etcd.GetDefaultLabels(), +// PublishNotReadyAddresses: true, +// Ports: []corev1.ServicePort{ +// { +// Name: "client", +// Protocol: corev1.ProtocolTCP, +// Port: *etcd.Spec.Etcd.ClientPort, +// TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ClientPort)), +// }, +// { +// Name: "server", +// Protocol: corev1.ProtocolTCP, +// Port: *etcd.Spec.Etcd.ServerPort, +// TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), +// }, +// { +// Name: "backuprestore", +// Protocol: corev1.ProtocolTCP, +// Port: *etcd.Spec.Backup.Port, +// TargetPort: intstr.FromInt(int(*etcd.Spec.Backup.Port)), +// }, +// }, +// }, +// } +//} +// +//func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl client.Client, op resource.Operator) { +// g = gomega.NewWithT(t) +// ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") +// etcd = sampleEtcd() +// if withExistingService { +// cl = fakeclient.NewClientBuilder().WithObjects(existingClientService(etcd)).Build() +// } else { +// +// cl = fakeclient.NewClientBuilder().Build() +// } +// op = New(cl) +// return +//} +// +//func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl *mockclient.MockClient, op resource.Operator) { +// g = gomega.NewWithT(t) +// ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") +// etcd = sampleEtcd() +// cl = mockclient.NewMockClient(gomock.NewController(t)) +// op = New(cl) +// return +//} diff --git a/test/sample/service.go b/test/sample/service.go new file mode 100644 index 000000000..3bdd4e96e --- /dev/null +++ b/test/sample/service.go @@ -0,0 +1,58 @@ +package sample + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + defaultBackupPort = 8080 + defaultClientPort = 2379 + defaultServerPort = 2380 +) + +// NewClientService creates a new sample client service initializing it from the passed in etcd object. +func NewClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetClientServiceName(), + Namespace: etcd.Namespace, + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + Spec: corev1.ServiceSpec{ + Ports: getClientServicePorts(etcd), + Selector: etcd.GetDefaultLabels(), + Type: corev1.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + }, + } +} + +func getClientServicePorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + return []corev1.ServicePort{ + { + Name: "client", + Protocol: corev1.ProtocolTCP, + Port: clientPort, + TargetPort: intstr.FromInt(int(clientPort)), + }, + { + Name: "server", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }, + { + Name: "backuprestore", + Protocol: corev1.ProtocolTCP, + Port: backupPort, + TargetPort: intstr.FromInt(int(backupPort)), + }, + } +} diff --git a/test/utils/fakeclient.go b/test/utils/fakeclient.go new file mode 100644 index 000000000..7330da523 --- /dev/null +++ b/test/utils/fakeclient.go @@ -0,0 +1,61 @@ +package utils + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// FakeClientBuilder builds a fake client with initial set of objects and additionally +// provides an ability to set custom errors for operations supported on the client. +type FakeClientBuilder struct { + clientBuilder *fakeclient.ClientBuilder + getErr error + createErr error + patchErr error +} + +// NewFakeClientBuilder creates a new FakeClientBuilder. +func NewFakeClientBuilder() *FakeClientBuilder { + return &FakeClientBuilder{ + clientBuilder: fakeclient.NewClientBuilder(), + } +} + +// WithObjects initializes the underline controller-runtime fake client with objects. +func (b *FakeClientBuilder) WithObjects(objs ...client.Object) *FakeClientBuilder { + b.clientBuilder.WithObjects(objs...) + return b +} + +// WithGetError sets the error that should be returned when a Get request is made on the fake client. +func (b *FakeClientBuilder) WithGetError(err error) *FakeClientBuilder { + b.getErr = err + return b +} + +// Build returns an instance of client.WithWatch which has capability to return the configured errors for operations. +func (b *FakeClientBuilder) Build() client.WithWatch { + return &fakeClient{ + WithWatch: b.clientBuilder.Build(), + getErr: b.getErr, + createErr: b.createErr, + patchErr: b.patchErr, + } +} + +type fakeClient struct { + client.WithWatch + getErr error + createErr error + patchErr error +} + +// Get overwrites the fake client Get implementation with a capability to return any configured error. +func (f *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if f.getErr != nil { + return f.getErr + } + return f.WithWatch.Get(ctx, key, obj, opts...) +} From 84d361dac22c6408bf7989dec1513ad49300d137 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 3 Jan 2024 16:30:06 +0530 Subject: [PATCH 044/235] removed logger from New functions --- internal/controller/etcd/reconciler.go | 12 ++++----- .../poddisruptionbudget.go | 5 +--- internal/operator/role/role.go | 5 +--- internal/operator/rolebinding/rolebinding.go | 7 ++--- .../operator/serviceaccount/serviceaccount.go | 7 ++--- .../operator/snapshotlease/snapshotlease.go | 9 +++---- internal/operator/statefulset/statefulset.go | 27 +++++++++---------- 7 files changed, 27 insertions(+), 45 deletions(-) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index ac3f8d6fe..4c2da9628 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -123,15 +123,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logger, config *Config, imageVector imagevector.ImageVector) operator.Registry { reg := operator.NewRegistry() reg.Register(operator.ConfigMapKind, configmap.New(client)) - reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, logger, config.DisableEtcdServiceAccountAutomount)) + reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, config.DisableEtcdServiceAccountAutomount)) reg.Register(operator.MemberLeaseKind, memberlease.New(client)) - reg.Register(operator.SnapshotLeaseKind, snapshotlease.New(client, logger)) + reg.Register(operator.SnapshotLeaseKind, snapshotlease.New(client)) reg.Register(operator.ClientServiceKind, clientservice.New(client)) reg.Register(operator.PeerServiceKind, peerservice.New(client)) - reg.Register(operator.PodDisruptionBudgetKind, poddistruptionbudget.New(client, logger)) - reg.Register(operator.RoleKind, role.New(client, logger)) - reg.Register(operator.RoleBindingKind, rolebinding.New(client, logger)) - reg.Register(operator.StatefulSetKind, statefulset.New(client, logger, imageVector, config.FeatureGates)) + reg.Register(operator.PodDisruptionBudgetKind, poddistruptionbudget.New(client)) + reg.Register(operator.RoleKind, role.New(client)) + reg.Register(operator.RoleBindingKind, rolebinding.New(client)) + reg.Register(operator.StatefulSetKind, statefulset.New(client, imageVector, config.FeatureGates)) return reg } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 591827395..cb80ada44 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -7,7 +7,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,7 +16,6 @@ import ( type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -55,10 +53,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return r.client.Delete(ctx, emptyPodDisruptionBudget(getObjectKey(etcd))) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 1599276e4..61101cf19 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -4,7 +4,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,7 +12,6 @@ import ( type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -44,10 +42,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return r.client.Delete(ctx, emptyRole(etcd)) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index 27ad7de4d..f216ee5cd 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -4,7 +4,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,7 +12,6 @@ import ( type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -59,7 +57,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } return nil }) - r.logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) + ctx.Logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) return err } @@ -67,10 +65,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 00ee867f9..1ccd88672 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -4,7 +4,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,7 +13,6 @@ import ( type _resource struct { client client.Client - logger logr.Logger disableAutoMount bool } @@ -39,7 +37,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) sa.AutomountServiceAccountToken = pointer.Bool(!r.disableAutoMount) return nil }) - r.logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) + ctx.Logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) return err } @@ -47,10 +45,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return client.IgnoreNotFound(r.client.Delete(ctx, emptyServiceAccount(getObjectKey(etcd)))) } -func New(client client.Client, logger logr.Logger, disableAutomount bool) resource.Operator { +func New(client client.Client, disableAutomount bool) resource.Operator { return &_resource{ client: client, - logger: logger, disableAutoMount: disableAutomount, } } diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 100f139a8..308fae840 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -9,7 +9,6 @@ import ( "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,7 +20,6 @@ const purpose = "etcd-snapshot-lease" type _resource struct { client client.Client - logger logr.Logger } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { @@ -58,7 +56,7 @@ func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*c func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { - r.logger.Info("Backup has been disabled. Triggering delete of snapshot leases") + ctx.Logger.Info("Backup has been disabled. Triggering delete of snapshot leases") return r.TriggerDelete(ctx, etcd) } for _, objKey := range getObjectKeys(etcd) { @@ -66,7 +64,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) if err != nil { return err } - r.logger.Info("Triggered create or update", "lease", objKey, "result", opResult) + ctx.Logger.Info("Triggered create or update", "lease", objKey, "result", opResult) } return nil } @@ -91,10 +89,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return nil } -func New(client client.Client, logger logr.Logger) resource.Operator { +func New(client client.Client) resource.Operator { return &_resource{ client: client, - logger: logger, } } diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index d14e3123e..2a4068185 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -12,7 +12,6 @@ import ( "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" - "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,15 +20,13 @@ import ( type _resource struct { client client.Client - logger logr.Logger imageVector imagevector.ImageVector useEtcdWrapper bool } -func New(client client.Client, logger logr.Logger, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) resource.Operator { +func New(client client.Client, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) resource.Operator { return &_resource{ client: client, - logger: logger, imageVector: imageVector, useEtcdWrapper: featureGates[features.UseEtcdWrapper], } @@ -79,7 +76,7 @@ func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *dr func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) mutatingFn := func() error { - if builder, err := newStsBuilder(r.client, r.logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { + if builder, err := newStsBuilder(r.client, ctx.Logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { return err } else { return builder.Build(ctx) @@ -90,7 +87,7 @@ func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alph return err } - r.logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) + ctx.Logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) return nil } @@ -158,7 +155,7 @@ func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alph func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { if existingSts.Generation > 1 && hasImmutableFieldChanged(existingSts, etcd) { - r.logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") + ctx.Logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") if err := r.TriggerDelete(ctx, etcd); err != nil { return fmt.Errorf("error deleting StatefulSet with immutable field updates: %v", err) @@ -175,19 +172,19 @@ func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) } func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) + peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, ctx.Logger) if err != nil { return fmt.Errorf("error checking if peer URL TLS is enabled: %v", err) } if isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { - r.logger.Info("Enabling TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) + ctx.Logger.Info("Enabling TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) if err := r.createOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %v", err) } - tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, r.logger) + tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, ctx.Logger) if err != nil { return fmt.Errorf("error verifying if TLS is enabled post-patch: %v", err) } @@ -195,11 +192,11 @@ func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *drui return fmt.Errorf("failed to enable TLS for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) } - if err := deleteAllStsPods(ctx, r.client, r.logger, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { + if err := deleteAllStsPods(ctx, r.client, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { return fmt.Errorf("error deleting StatefulSet pods after enabling TLS: %v", err) } - r.logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) + ctx.Logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) } return nil @@ -210,7 +207,7 @@ func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druid return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } -func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger logr.Logger, opName string, sts *appsv1.StatefulSet) error { +func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, opName string, sts *appsv1.StatefulSet) error { // Get all Pods belonging to the StatefulSet podList := &corev1.PodList{} listOpts := []client.ListOption{ @@ -219,13 +216,13 @@ func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger log } if err := cl.List(ctx, podList, listOpts...); err != nil { - logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) + ctx.Logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) return err } for _, pod := range podList.Items { if err := cl.Delete(ctx, &pod); err != nil { - logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) + ctx.Logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) return err } } From b5a225aa027ce6c5231b9c542e6e4bb2df47b2cd Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Thu, 4 Jan 2024 12:16:32 +0530 Subject: [PATCH 045/235] Added the ObservedGeneration check to E2E test --- test/e2e/etcd_backup_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index a9a74f012..3fb5fa40f 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -221,6 +221,11 @@ func checkEtcdReady(ctx context.Context, cl client.Client, logger logr.Logger, e return err } + // Ensure the etcd cluster's current generation matches the observed generation + if etcd.Status.ObservedGeneration != &etcd.Generation { + return fmt.Errorf("etcd '%s' is not at the expected generation (observed: %d, expected: %d)", etcd.Name, etcd.Status.ObservedGeneration, etcd.Generation) + } + if etcd.Status.Ready == nil || *etcd.Status.Ready != true { return fmt.Errorf("etcd %s is not ready", etcd.Name) } From 452c62a63aa14b4033e5bed1d7540aa6c469b953 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 11:16:45 +0530 Subject: [PATCH 046/235] Add delete error handling in FakeClientBuilder --- test/utils/fakeclient.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/utils/fakeclient.go b/test/utils/fakeclient.go index 7330da523..8cb2acf28 100644 --- a/test/utils/fakeclient.go +++ b/test/utils/fakeclient.go @@ -14,6 +14,7 @@ type FakeClientBuilder struct { getErr error createErr error patchErr error + deleteErr error } // NewFakeClientBuilder creates a new FakeClientBuilder. @@ -35,6 +36,12 @@ func (b *FakeClientBuilder) WithGetError(err error) *FakeClientBuilder { return b } +// WithDeleteError sets the error that should be returned when a Delete request is made on the fake client. +func (b *FakeClientBuilder) WithDeleteError(err error) *FakeClientBuilder { + b.deleteErr = err + return b +} + // Build returns an instance of client.WithWatch which has capability to return the configured errors for operations. func (b *FakeClientBuilder) Build() client.WithWatch { return &fakeClient{ @@ -42,6 +49,7 @@ func (b *FakeClientBuilder) Build() client.WithWatch { getErr: b.getErr, createErr: b.createErr, patchErr: b.patchErr, + deleteErr: b.deleteErr, } } @@ -50,6 +58,7 @@ type fakeClient struct { getErr error createErr error patchErr error + deleteErr error } // Get overwrites the fake client Get implementation with a capability to return any configured error. @@ -59,3 +68,11 @@ func (f *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.O } return f.WithWatch.Get(ctx, key, obj, opts...) } + +// Delete overwrites the fake client Get implementation with a capability to return any configured error. +func (f *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if f.deleteErr != nil { + return f.deleteErr + } + return f.WithWatch.Delete(ctx, obj, opts...) +} From d62db5e90343a62af2f469302ec5d912905aec3a Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 11:18:13 +0530 Subject: [PATCH 047/235] Refactor clientservice tests --- .../clientservice/clientservice_test.go | 460 ++++++++---------- 1 file changed, 213 insertions(+), 247 deletions(-) diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 233e8d97b..53760dab2 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -17,29 +17,36 @@ package clientservice import ( "context" "errors" + "fmt" "testing" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" + "github.com/onsi/gomega" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// ------------------------ GetExistingResourceNames ------------------------ - const ( testEtcdName = "test-etcd" testNs = "test-namespace" ) +// ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { internalErr := errors.New("test internal error") etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() - t.Parallel() testcases := []struct { name string svcExists bool @@ -71,6 +78,7 @@ func TestGetExistingResourceNames(t *testing.T) { } g := NewWithT(t) + t.Parallel() for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -94,247 +102,205 @@ func TestGetExistingResourceNames(t *testing.T) { } } -//func TestClientServiceGetExistingResourceNames_WithExistingService(t *testing.T) { -// g, ctx, etcd, _, op := setupWithFakeClient(t, true) -// -// err := op.Sync(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// -// names, err := op.GetExistingResourceNames(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// g.Expect(names).To(gomega.ContainElement(etcd.GetClientServiceName())) -//} -// -//func TestClientServiceGetExistingResourceNames_ServiceNotFound(t *testing.T) { -// g, ctx, etcd, _, op := setupWithFakeClient(t, false) -// -// names, err := op.GetExistingResourceNames(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// g.Expect(names).To(gomega.BeEmpty()) -//} -// -//func TestClientServiceGetExistingResourceNames_WithError(t *testing.T) { -// g, ctx, etcd, cl, op := setupWithMockClient(t) -// cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() -// -// names, err := op.GetExistingResourceNames(ctx, etcd) -// g.Expect(err).To(gomega.HaveOccurred()) -// g.Expect(names).To(gomega.BeEmpty()) -//} -// -//// ----------------------------------- Sync ----------------------------------- -// -//func TestClientServiceSync_CreateNewService(t *testing.T) { -// g, ctx, etcd, cl, op := setupWithFakeClient(t, false) -// err := op.Sync(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// -// service := &corev1.Service{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: etcd.GetClientServiceName(), -// Namespace: etcd.Namespace, -// }, -// } -// err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// checkClientService(g, service, etcd) -//} -// -//func TestClientServiceSync_UpdateExistingService(t *testing.T) { -// g, ctx, etcd, cl, op := setupWithFakeClient(t, true) -// etcd.Spec.Etcd.ServerPort = nil -// etcd.Spec.Etcd.ClientPort = nil -// etcd.Spec.Backup.Port = nil -// -// etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{ -// Labels: map[string]string{"testingKey": "testingValue"}, -// Annotations: map[string]string{"testingAnnotationKey": "testingValue"}, -// } -// -// err := op.Sync(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// -// service := &corev1.Service{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: etcd.GetClientServiceName(), -// Namespace: etcd.Namespace, -// }, -// } -// err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// g.Expect(service).ToNot(gomega.BeNil()) -// checkClientService(g, service, etcd) -//} -// -//func TestClientServiceSync_WithClientError(t *testing.T) { -// g, ctx, etcd, cl, op := setupWithMockClient(t) -// cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() -// cl.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake create error")).AnyTimes() -// cl.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake update error")).AnyTimes() -// -// err := op.Sync(ctx, etcd) -// g.Expect(err).To(gomega.HaveOccurred()) -// g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrSyncingClientService))) -//} -// -//// ----------------------------- TriggerDelete ------------------------------- -// -//func TestClientServiceTriggerDelete_ExistingService(t *testing.T) { -// g, ctx, etcd, cl, op := setupWithFakeClient(t, true) -// err := op.TriggerDelete(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// -// serviceList := &corev1.List{} -// err = cl.List(ctx, serviceList) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -// g.Expect(serviceList.Items).To(gomega.BeEmpty()) -//} -// -//func TestClientServiceTriggerDelete_ServiceNotFound(t *testing.T) { -// g, ctx, etcd, _, op := setupWithFakeClient(t, false) -// -// err := op.TriggerDelete(ctx, etcd) -// g.Expect(err).NotTo(gomega.HaveOccurred()) -//} -// -//func TestClientServiceTriggerDelete_WithClientError(t *testing.T) { -// g, ctx, etcd, cl, op := setupWithMockClient(t) -// // Configure the mock client to return an error on delete operation -// cl.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake delete error")).AnyTimes() -// -// err := op.TriggerDelete(ctx, etcd) -// g.Expect(err).To(gomega.HaveOccurred()) -// g.Expect(err.Error()).To(gomega.ContainSubstring(string(ErrDeletingClientService))) -//} -// -//// ---------------------------- Helper Functions ----------------------------- -// -//func sampleEtcd() *druidv1alpha1.Etcd { -// return &druidv1alpha1.Etcd{ -// -// ObjectMeta: metav1.ObjectMeta{ -// Name: "test-etcd", -// Namespace: "test-namespace", -// UID: "xxx-yyy-zzz", -// }, -// Spec: druidv1alpha1.EtcdSpec{ -// -// Backup: druidv1alpha1.BackupSpec{ -// Port: pointer.Int32(1111), -// }, -// Etcd: druidv1alpha1.EtcdConfig{ -// ClientPort: pointer.Int32(2222), -// ServerPort: pointer.Int32(3333), -// }, -// }, -// } -//} -// -//func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { -// clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) -// backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) -// peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) -// -// expectedLabels := etcd.GetDefaultLabels() -// var expectedAnnotations map[string]string -// if etcd.Spec.Etcd.ClientService != nil { -// expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations -// expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) -// } -// g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) -// g.Expect(svc.Annotations).To(gomega.Equal(expectedAnnotations)) -// g.Expect(svc.Labels).To(gomega.Equal(expectedLabels)) -// g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) -// g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) -// g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) -// g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( -// gomega.Equal(corev1.ServicePort{ -// Name: "client", -// Protocol: corev1.ProtocolTCP, -// Port: clientPort, -// TargetPort: intstr.FromInt(int(clientPort)), -// }), -// gomega.Equal(corev1.ServicePort{ -// Name: "server", -// Protocol: corev1.ProtocolTCP, -// Port: peerPort, -// TargetPort: intstr.FromInt(int(peerPort)), -// }), -// gomega.Equal(corev1.ServicePort{ -// Name: "backuprestore", -// Protocol: corev1.ProtocolTCP, -// Port: backupPort, -// TargetPort: intstr.FromInt(int(backupPort)), -// }), -// )) -//} -// -//func existingClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { -// var annotations, labels map[string]string -// if etcd.Spec.Etcd.ClientService != nil { -// annotations = etcd.Spec.Etcd.ClientService.Annotations -// labels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) -// -// } -// return &corev1.Service{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: etcd.GetClientServiceName(), -// Namespace: etcd.Namespace, -// Annotations: annotations, -// Labels: labels, -// OwnerReferences: []metav1.OwnerReference{ -// etcd.GetAsOwnerReference(), -// }, -// }, -// Spec: corev1.ServiceSpec{ -// Type: corev1.ServiceTypeClusterIP, -// ClusterIP: corev1.ClusterIPNone, -// SessionAffinity: corev1.ServiceAffinityNone, -// Selector: etcd.GetDefaultLabels(), -// PublishNotReadyAddresses: true, -// Ports: []corev1.ServicePort{ -// { -// Name: "client", -// Protocol: corev1.ProtocolTCP, -// Port: *etcd.Spec.Etcd.ClientPort, -// TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ClientPort)), -// }, -// { -// Name: "server", -// Protocol: corev1.ProtocolTCP, -// Port: *etcd.Spec.Etcd.ServerPort, -// TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), -// }, -// { -// Name: "backuprestore", -// Protocol: corev1.ProtocolTCP, -// Port: *etcd.Spec.Backup.Port, -// TargetPort: intstr.FromInt(int(*etcd.Spec.Backup.Port)), -// }, -// }, -// }, -// } -//} -// -//func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl client.Client, op resource.Operator) { -// g = gomega.NewWithT(t) -// ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") -// etcd = sampleEtcd() -// if withExistingService { -// cl = fakeclient.NewClientBuilder().WithObjects(existingClientService(etcd)).Build() -// } else { -// -// cl = fakeclient.NewClientBuilder().Build() -// } -// op = New(cl) -// return -//} -// -//func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl *mockclient.MockClient, op resource.Operator) { -// g = gomega.NewWithT(t) -// ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") -// etcd = sampleEtcd() -// cl = mockclient.NewMockClient(gomock.NewController(t)) -// op = New(cl) -// return -//} +// ----------------------------------- Sync ----------------------------------- +func TestClientServiceSync(t *testing.T) { + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + svcExists bool + setupFn func(eb *testutils.EtcdBuilder) + expectError *druiderr.DruidError + getErr *apierrors.StatusError + }{ + { + name: "Create new service", + svcExists: false, + }, + { + name: "Update existing service", + svcExists: true, + + setupFn: func(eb *testutils.EtcdBuilder) { + eb.WithEtcdClientPort(nil). + WithBackupPort(nil). + WithEtcdServerPort(nil). + WithEtcdClientServiceLabels(map[string]string{"testingKey": "testingValue"}). + WithEtcdClientServiceAnnotations(map[string]string{"testingAnnotationKey": "testingValue"}) + }, + }, + { + name: "With client error", + svcExists: false, + setupFn: nil, + expectError: &druiderr.DruidError{ + Code: ErrSyncingClientService, + Cause: fmt.Errorf("Fake get error"), + Operation: "Sync", + Message: "Error during create or update of client service", + }, + getErr: apierrors.NewInternalError(errors.New("Fake get error")), + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.setupFn != nil { + tc.setupFn(etcdBuilder) + } + etcd := etcdBuilder.Build() + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.svcExists { + fakeClientBuilder.WithObjects(sample.NewClientService(testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, etcd) + if tc.expectError != nil { + g.Expect(err).To(gomega.HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.getErr)).To(BeTrue()) + g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) + g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) + + } else { + g.Expect(err).NotTo(gomega.HaveOccurred()) + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetClientServiceName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(opCtx, client.ObjectKeyFromObject(service), service) + g.Expect(err).NotTo(gomega.HaveOccurred()) + checkClientService(g, service, etcd) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestClientServiceTriggerDelete(t *testing.T) { + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + svcExists bool + setupFn func(eb *testutils.EtcdBuilder) + expectError *druiderr.DruidError + deleteErr *apierrors.StatusError + }{ + { + name: "Existing Service - Delete Operation", + svcExists: false, + }, + { + name: "Service Not Found - No Operation", + svcExists: true, + setupFn: func(eb *testutils.EtcdBuilder) { + eb.WithEtcdClientPort(nil). + WithBackupPort(nil). + WithEtcdServerPort(nil). + WithEtcdClientServiceLabels(map[string]string{"testingKey": "testingValue"}). + WithEtcdClientServiceAnnotations(map[string]string{"testingAnnotationKey": "testingValue"}) + }, + }, + { + name: "Client Error on Delete - Returns Error", + svcExists: true, + setupFn: nil, + expectError: &druiderr.DruidError{ + Code: ErrDeletingClientService, + Cause: errors.New("Fake delete error"), + Operation: "TriggerDelete", + Message: "Failed to delete client service", + }, + deleteErr: apierrors.NewInternalError(errors.New("Fake delete error")), + }, + } + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.setupFn != nil { + tc.setupFn(etcdBuilder) + } + etcd := etcdBuilder.Build() + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.deleteErr != nil { + fakeClientBuilder.WithDeleteError(tc.deleteErr) + } + if tc.svcExists { + fakeClientBuilder.WithObjects(sample.NewClientService(testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.TriggerDelete(opCtx, etcd) + if tc.expectError != nil { + g.Expect(err).To(gomega.HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.deleteErr)).To(BeTrue()) + g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) + g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) + + } else { + g.Expect(err).NotTo(gomega.HaveOccurred()) + serviceList := &corev1.List{} + err = cl.List(opCtx, serviceList) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(serviceList.Items).To(gomega.BeEmpty()) + + } + }) + } +} + +func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + + expectedLabels := etcd.GetDefaultLabels() + var expectedAnnotations map[string]string + if etcd.Spec.Etcd.ClientService != nil { + expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations + expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) + } + g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(svc.Annotations).To(gomega.Equal(expectedAnnotations)) + g.Expect(svc.Labels).To(gomega.Equal(expectedLabels)) + g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) + g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) + g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( + gomega.Equal(corev1.ServicePort{ + Name: "client", + Protocol: corev1.ProtocolTCP, + Port: clientPort, + TargetPort: intstr.FromInt(int(clientPort)), + }), + gomega.Equal(corev1.ServicePort{ + Name: "server", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }), + gomega.Equal(corev1.ServicePort{ + Name: "backuprestore", + Protocol: corev1.ProtocolTCP, + Port: backupPort, + TargetPort: intstr.FromInt(int(backupPort)), + }), + )) +} From aa9f48482d3ee04cb05b41cebc1de611452d933a Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 11:19:49 +0530 Subject: [PATCH 048/235] Extend EtcdBuilder with new configuration methods --- test/utils/etcd.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 31713d5b7..023f904e8 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -10,6 +10,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" "sigs.k8s.io/controller-runtime/pkg/client" corev1 "k8s.io/api/core/v1" @@ -228,6 +229,56 @@ func (eb *EtcdBuilder) WithProviderLocal() *EtcdBuilder { return eb } +func (eb *EtcdBuilder) WithEtcdClientPort(clientPort *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Etcd.ClientPort = clientPort + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + if eb.etcd.Spec.Etcd.ClientService == nil { + eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} + } + + eb.etcd.Spec.Etcd.ClientService.Labels = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]string) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + if eb.etcd.Spec.Etcd.ClientService == nil { + eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} + } + + eb.etcd.Spec.Etcd.ClientService.Annotations = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) + return eb +} + +func (eb *EtcdBuilder) WithEtcdServerPort(serverPort *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Etcd.ServerPort = serverPort + return eb +} + +func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Port = port + return eb +} + func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { return eb.etcd } From 772203e1b25e748e7a3cf38fae3d0cbee4a8e373 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 11:35:09 +0530 Subject: [PATCH 049/235] EtcdBuilder moved to test/sample from test/utils --- pkg/utils/image_test.go | 12 +- pkg/utils/statefulset_test.go | 3 +- .../controllers/compaction/reconciler_test.go | 11 +- .../controllers/custodian/reconciler_test.go | 7 +- .../controllers/etcd/reconciler_test.go | 13 +- .../controllers/secret/reconciler_test.go | 3 +- test/sample/etcd.go | 430 +++++++++++++++++- test/utils/etcd.go | 406 ----------------- test/utils/misc.go | 5 +- 9 files changed, 459 insertions(+), 431 deletions(-) diff --git a/pkg/utils/image_test.go b/pkg/utils/image_test.go index e0f61a336..9c0109d3b 100644 --- a/pkg/utils/image_test.go +++ b/pkg/utils/image_test.go @@ -7,7 +7,7 @@ package utils import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" - testutils "github.com/gardener/etcd-druid/test/utils" + testsample "github.com/gardener/etcd-druid/test/sample" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/ginkgo/v2" @@ -28,7 +28,7 @@ var _ = Describe("Image retrieval tests", func() { ) It("etcd spec defines etcd and backup-restore images", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() imageVector = createImageVector(true, true, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) Expect(err).To(BeNil()) @@ -42,7 +42,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec has no image defined and image vector has both images set", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil @@ -63,7 +63,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil imageVector = createImageVector(true, false, false, false) @@ -81,7 +81,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("both spec and image vector do not have backup-restore image", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Backup.Image = nil imageVector = createImageVector(true, false, false, false) @@ -93,7 +93,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec has no images defined, image vector has all images, and UseEtcdWrapper feature gate is turned on", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil diff --git a/pkg/utils/statefulset_test.go b/pkg/utils/statefulset_test.go index 1f489680c..a9c69f3c0 100644 --- a/pkg/utils/statefulset_test.go +++ b/pkg/utils/statefulset_test.go @@ -9,6 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/client/kubernetes" + testsample "github.com/gardener/etcd-druid/test/sample" "github.com/gardener/etcd-druid/test/utils" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -74,7 +75,7 @@ var _ = Describe("tests for statefulset utility functions", func() { BeforeEach(func() { ctx = context.TODO() stsListToCleanup = &appsv1.StatefulSetList{} - etcd = utils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() }) AfterEach(func() { diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index 7e3a4db42..d9842a0ee 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -12,6 +12,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" "github.com/gardener/etcd-druid/pkg/utils" + testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -47,7 +48,7 @@ var _ = Describe("Compaction Controller", func() { j *batchv1.Job ) - instance = testutils.EtcdBuilderWithDefaults(name, namespace).WithTLS().WithStorageProvider(provider).Build() + instance = testsample.EtcdBuilderWithDefaults(name, namespace).WithTLS().WithStorageProvider(provider).Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -102,7 +103,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testutils.EtcdBuilderWithDefaults("foo77", namespace).WithProviderLocal().Build() + instance = testsample.EtcdBuilderWithDefaults("foo77", namespace).WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -154,7 +155,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testutils.EtcdBuilderWithDefaults("foo78", "default").WithProviderLocal().Build() + instance = testsample.EtcdBuilderWithDefaults("foo78", "default").WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -198,7 +199,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testutils.EtcdBuilderWithDefaults("foo79", "default").WithProviderLocal().Build() + instance = testsample.EtcdBuilderWithDefaults("foo79", "default").WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -239,7 +240,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testutils.EtcdBuilderWithDefaults("foo80", "default").WithProviderLocal().Build() + instance = testsample.EtcdBuilderWithDefaults("foo80", "default").WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running diff --git a/test/integration/controllers/custodian/reconciler_test.go b/test/integration/controllers/custodian/reconciler_test.go index fb1f4ab1a..c4fcb0007 100644 --- a/test/integration/controllers/custodian/reconciler_test.go +++ b/test/integration/controllers/custodian/reconciler_test.go @@ -11,6 +11,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" componentpdb "github.com/gardener/etcd-druid/pkg/component/etcd/poddisruptionbudget" + testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -38,7 +39,7 @@ var _ = Describe("Custodian Controller", func() { ) BeforeEach(func() { - instance = testutils.EtcdBuilderWithDefaults(name, namespace).Build() + instance = testsample.EtcdBuilderWithDefaults(name, namespace).Build() Expect(k8sClient.Create(ctx, instance)).To(Succeed()) // wait for Etcd creation to succeed @@ -164,7 +165,7 @@ var _ = Describe("Custodian Controller", func() { Describe("PodDisruptionBudget", func() { Context("minAvailable of PodDisruptionBudget", func() { When("having a single node cluster", func() { - etcd := testutils.EtcdBuilderWithDefaults("test", "default").WithReadyStatus().Build() + etcd := testsample.EtcdBuilderWithDefaults("test", "default").WithReadyStatus().Build() Expect(len(etcd.Status.Members)).To(BeEquivalentTo(1)) @@ -177,7 +178,7 @@ var _ = Describe("Custodian Controller", func() { }) When("having a multi node cluster", func() { - etcd := testutils.EtcdBuilderWithDefaults("test", "default").WithReplicas(3).WithReadyStatus().Build() + etcd := testsample.EtcdBuilderWithDefaults("test", "default").WithReplicas(3).WithReadyStatus().Build() Expect(len(etcd.Status.Members)).To(BeEquivalentTo(3)) diff --git a/test/integration/controllers/etcd/reconciler_test.go b/test/integration/controllers/etcd/reconciler_test.go index 714863a35..fbabccae2 100644 --- a/test/integration/controllers/etcd/reconciler_test.go +++ b/test/integration/controllers/etcd/reconciler_test.go @@ -16,6 +16,7 @@ import ( "github.com/gardener/etcd-druid/pkg/common" "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/etcd-druid/test/integration/controllers/assets" + testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" gardenerUtils "github.com/gardener/gardener/pkg/utils" @@ -79,7 +80,7 @@ var _ = Describe("Etcd Controller", func() { ) BeforeEach(func() { - instance = testutils.EtcdBuilderWithDefaults("foo1", namespace).Build() + instance = testsample.EtcdBuilderWithDefaults("foo1", namespace).Build() storeSecret := instance.Spec.Backup.Store.SecretRef.Name errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) @@ -196,12 +197,12 @@ var _ = Describe("Etcd Controller", func() { rb *rbac.RoleBinding ctx = context.TODO() instance *druidv1alpha1.Etcd - instanceBuilder *testutils.EtcdBuilder + instanceBuilder *testsample.EtcdBuilder ) if etcdWithDefaults { - instanceBuilder = testutils.EtcdBuilderWithDefaults(etcdName, namespace) + instanceBuilder = testsample.EtcdBuilderWithDefaults(etcdName, namespace) } else { - instanceBuilder = testutils.EtcdBuilderWithoutDefaults(etcdName, namespace) + instanceBuilder = testsample.EtcdBuilderWithoutDefaults(etcdName, namespace) } if withTLS { instanceBuilder.WithTLS() @@ -278,7 +279,7 @@ var _ = Describe("Multinode ETCD", func() { ) BeforeEach(func() { - instance = testutils.EtcdBuilderWithDefaults("foo82", namespace).Build() + instance = testsample.EtcdBuilderWithDefaults("foo82", namespace).Build() storeSecret := instance.Spec.Backup.Store.SecretRef.Name errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) Expect(len(errors)).Should(BeZero()) @@ -379,7 +380,7 @@ var _ = Describe("Multinode ETCD", func() { ctx = context.TODO() instance *druidv1alpha1.Etcd ) - instance = testutils.EtcdBuilderWithDefaults(etcdName, namespace).WithReplicas(int32(replicas)).Build() + instance = testsample.EtcdBuilderWithDefaults(etcdName, namespace).WithReplicas(int32(replicas)).Build() if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { storeSecret := instance.Spec.Backup.Store.SecretRef.Name diff --git a/test/integration/controllers/secret/reconciler_test.go b/test/integration/controllers/secret/reconciler_test.go index 0082315c2..a3ac80a10 100644 --- a/test/integration/controllers/secret/reconciler_test.go +++ b/test/integration/controllers/secret/reconciler_test.go @@ -10,6 +10,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" + testsample "github.com/gardener/etcd-druid/test/sample" "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -32,7 +33,7 @@ var _ = Describe("Secret Controller", func() { ) BeforeEach(func() { - etcd = utils.EtcdBuilderWithDefaults("etcd", namespace).WithTLS().Build() + etcd = testsample.EtcdBuilderWithDefaults("etcd", namespace).WithTLS().Build() }) It("should reconcile the finalizers for the referenced secrets", func() { diff --git a/test/sample/etcd.go b/test/sample/etcd.go index 28f85fcd9..4add6d27e 100644 --- a/test/sample/etcd.go +++ b/test/sample/etcd.go @@ -1,3 +1,431 @@ +// Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sample -// Add the etcd builder code from test/utils/etcd.go here +import ( + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" + testutils "github.com/gardener/etcd-druid/test/utils" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" +) + +var ( + deltaSnapshotPeriod = metav1.Duration{ + Duration: 300 * time.Second, + } + garbageCollectionPeriod = metav1.Duration{ + Duration: 43200 * time.Second, + } + clientPort int32 = 2379 + serverPort int32 = 2380 + backupPort int32 = 8080 + imageEtcd = "eu.gcr.io/gardener-project/gardener/etcd-wrapper:v0.1.0" + imageBR = "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.25.0" + snapshotSchedule = "0 */24 * * *" + defragSchedule = "0 */24 * * *" + container = "default.bkp" + storageCapacity = resource.MustParse("5Gi") + storageClass = "default" + priorityClassName = "class_priority" + deltaSnapShotMemLimit = resource.MustParse("100Mi") + autoCompactionMode = druidv1alpha1.Periodic + autoCompactionRetention = "2m" + quota = resource.MustParse("8Gi") + localProvider = druidv1alpha1.StorageProvider("Local") + prefix = "/tmp" + uid = "a9b8c7d6e5f4" + volumeClaimTemplateName = "etcd-main" + garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) + metricsBasic = druidv1alpha1.Basic + etcdSnapshotTimeout = metav1.Duration{ + Duration: 10 * time.Minute, + } + etcdDefragTimeout = metav1.Duration{ + Duration: 10 * time.Minute, + } +) + +type EtcdBuilder struct { + etcd *druidv1alpha1.Etcd +} + +func EtcdBuilderWithDefaults(name, namespace string) *EtcdBuilder { + builder := EtcdBuilder{} + builder.etcd = getDefaultEtcd(name, namespace) + return &builder +} + +func EtcdBuilderWithoutDefaults(name, namespace string) *EtcdBuilder { + builder := EtcdBuilder{} + builder.etcd = getEtcdWithoutDefaults(name, namespace) + return &builder +} + +func (eb *EtcdBuilder) WithReplicas(replicas int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Replicas = replicas + return eb +} + +func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + clientTLSConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, + DataKey: pointer.String("ca.crt"), + }, + ClientTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-client-tls", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-server-tls", + }, + } + + peerTLSConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "peer-url-ca-etcd", + }, + DataKey: pointer.String("ca.crt"), + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", + }, + } + + eb.etcd.Spec.Etcd.ClientUrlTLS = clientTLSConfig + eb.etcd.Spec.Etcd.PeerUrlTLS = peerTLSConfig + eb.etcd.Spec.Backup.TLS = clientTLSConfig + + return eb +} + +func (eb *EtcdBuilder) WithReadyStatus() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + members := make([]druidv1alpha1.EtcdMemberStatus, 0) + for i := 0; i < int(eb.etcd.Spec.Replicas); i++ { + members = append(members, druidv1alpha1.EtcdMemberStatus{Status: druidv1alpha1.EtcdMemberStatusReady}) + } + eb.etcd.Status = druidv1alpha1.EtcdStatus{ + ReadyReplicas: eb.etcd.Spec.Replicas, + Replicas: eb.etcd.Spec.Replicas, + CurrentReplicas: eb.etcd.Spec.Replicas, + UpdatedReplicas: eb.etcd.Spec.Replicas, + Ready: pointer.Bool(true), + Members: members, + Conditions: []druidv1alpha1.Condition{ + {Type: druidv1alpha1.ConditionTypeAllMembersReady, Status: druidv1alpha1.ConditionTrue}, + }, + } + + return eb +} + +func (eb *EtcdBuilder) WithStorageProvider(provider druidv1alpha1.StorageProvider) *EtcdBuilder { + // TODO: there is no default case right now which is not very right, returning an error in a default case makes it difficult to chain + // This should be improved later + switch provider { + case "aws": + return eb.WithProviderS3() + case "azure": + return eb.WithProviderABS() + case "alicloud": + return eb.WithProviderOSS() + case "gcp": + return eb.WithProviderGCS() + case "openstack": + return eb.WithProviderSwift() + case "local": + return eb.WithProviderLocal() + default: + return eb + } +} + +func (eb *EtcdBuilder) WithProviderS3() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "aws", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderABS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "azure", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderGCS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "gcp", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderSwift() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "openstack", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderOSS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "alicloud", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderLocal() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStoreForLocal( + eb.etcd.Name, + ) + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientPort(clientPort *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Etcd.ClientPort = clientPort + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + if eb.etcd.Spec.Etcd.ClientService == nil { + eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} + } + + eb.etcd.Spec.Etcd.ClientService.Labels = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]string) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + if eb.etcd.Spec.Etcd.ClientService == nil { + eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} + } + + eb.etcd.Spec.Etcd.ClientService.Annotations = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) + return eb +} + +func (eb *EtcdBuilder) WithEtcdServerPort(serverPort *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Etcd.ServerPort = serverPort + return eb +} + +func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Port = port + return eb +} + +func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { + return eb.etcd +} + +func getEtcdWithoutDefaults(name, namespace string) *druidv1alpha1.Etcd { + return &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: druidv1alpha1.EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + "role": "test", + }, + Labels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + }, + Replicas: 1, + Backup: druidv1alpha1.BackupSpec{}, + Etcd: druidv1alpha1.EtcdConfig{}, + Common: druidv1alpha1.SharedConfig{}, + }, + } +} + +func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { + return &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID(uid), + }, + Spec: druidv1alpha1.EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + "role": "test", + }, + Labels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + }, + Replicas: 1, + StorageCapacity: &storageCapacity, + StorageClass: &storageClass, + PriorityClassName: &priorityClassName, + VolumeClaimTemplate: &volumeClaimTemplateName, + Backup: druidv1alpha1.BackupSpec{ + Image: &imageBR, + Port: &backupPort, + FullSnapshotSchedule: &snapshotSchedule, + GarbageCollectionPolicy: &garbageCollectionPolicy, + GarbageCollectionPeriod: &garbageCollectionPeriod, + DeltaSnapshotPeriod: &deltaSnapshotPeriod, + DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, + EtcdSnapshotTimeout: &etcdSnapshotTimeout, + + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": testutils.ParseQuantity("500m"), + "memory": testutils.ParseQuantity("2Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": testutils.ParseQuantity("23m"), + "memory": testutils.ParseQuantity("128Mi"), + }, + }, + Store: &druidv1alpha1.StoreSpec{ + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + Container: &container, + Provider: &localProvider, + Prefix: prefix, + }, + }, + Etcd: druidv1alpha1.EtcdConfig{ + Quota: "a, + Metrics: &metricsBasic, + Image: &imageEtcd, + DefragmentationSchedule: &defragSchedule, + EtcdDefragTimeout: &etcdDefragTimeout, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": testutils.ParseQuantity("2500m"), + "memory": testutils.ParseQuantity("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": testutils.ParseQuantity("500m"), + "memory": testutils.ParseQuantity("1000Mi"), + }, + }, + ClientPort: &clientPort, + ServerPort: &serverPort, + }, + Common: druidv1alpha1.SharedConfig{ + AutoCompactionMode: &autoCompactionMode, + AutoCompactionRetention: &autoCompactionRetention, + }, + }, + } +} + +func getBackupStore(name string, provider druidv1alpha1.StorageProvider) *druidv1alpha1.StoreSpec { + return &druidv1alpha1.StoreSpec{ + Container: &container, + Prefix: name, + Provider: &provider, + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + } +} + +func getBackupStoreForLocal(name string) *druidv1alpha1.StoreSpec { + provider := druidv1alpha1.StorageProvider("local") + return &druidv1alpha1.StoreSpec{ + Container: &container, + Prefix: name, + Provider: &provider, + } +} diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 023f904e8..061affcfe 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -10,419 +10,13 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" "sigs.k8s.io/controller-runtime/pkg/client" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" ) -var ( - deltaSnapshotPeriod = metav1.Duration{ - Duration: 300 * time.Second, - } - garbageCollectionPeriod = metav1.Duration{ - Duration: 43200 * time.Second, - } - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - imageEtcd = "europe-docker.pkg.dev/gardener-project/public/gardener/etcd-wrapper:v0.1.0" - imageBR = "europe-docker.pkg.dev/gardener-project/public/gardener/etcdbrctl:v0.25.0" - snapshotSchedule = "0 */24 * * *" - defragSchedule = "0 */24 * * *" - container = "default.bkp" - storageCapacity = resource.MustParse("5Gi") - storageClass = "default" - priorityClassName = "class_priority" - deltaSnapShotMemLimit = resource.MustParse("100Mi") - autoCompactionMode = druidv1alpha1.Periodic - autoCompactionRetention = "2m" - quota = resource.MustParse("8Gi") - localProvider = druidv1alpha1.StorageProvider("Local") - prefix = "/tmp" - uid = "a9b8c7d6e5f4" - volumeClaimTemplateName = "etcd-main" - garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) - metricsBasic = druidv1alpha1.Basic - etcdSnapshotTimeout = metav1.Duration{ - Duration: 10 * time.Minute, - } - etcdDefragTimeout = metav1.Duration{ - Duration: 10 * time.Minute, - } -) - -type EtcdBuilder struct { - etcd *druidv1alpha1.Etcd -} - -func EtcdBuilderWithDefaults(name, namespace string) *EtcdBuilder { - builder := EtcdBuilder{} - builder.etcd = getDefaultEtcd(name, namespace) - return &builder -} - -func EtcdBuilderWithoutDefaults(name, namespace string) *EtcdBuilder { - builder := EtcdBuilder{} - builder.etcd = getEtcdWithoutDefaults(name, namespace) - return &builder -} - -func (eb *EtcdBuilder) WithReplicas(replicas int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Replicas = replicas - return eb -} - -func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - clientTLSConfig := &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "client-url-ca-etcd", - }, - DataKey: pointer.String("ca.crt"), - }, - ClientTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-client-tls", - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-server-tls", - }, - } - - peerTLSConfig := &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "peer-url-ca-etcd", - }, - DataKey: pointer.String("ca.crt"), - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "peer-url-etcd-server-tls", - }, - } - - eb.etcd.Spec.Etcd.ClientUrlTLS = clientTLSConfig - eb.etcd.Spec.Etcd.PeerUrlTLS = peerTLSConfig - eb.etcd.Spec.Backup.TLS = clientTLSConfig - - return eb -} - -func (eb *EtcdBuilder) WithReadyStatus() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - - members := make([]druidv1alpha1.EtcdMemberStatus, 0) - for i := 0; i < int(eb.etcd.Spec.Replicas); i++ { - members = append(members, druidv1alpha1.EtcdMemberStatus{Status: druidv1alpha1.EtcdMemberStatusReady}) - } - eb.etcd.Status = druidv1alpha1.EtcdStatus{ - ReadyReplicas: eb.etcd.Spec.Replicas, - Replicas: eb.etcd.Spec.Replicas, - CurrentReplicas: eb.etcd.Spec.Replicas, - UpdatedReplicas: eb.etcd.Spec.Replicas, - Ready: pointer.Bool(true), - Members: members, - Conditions: []druidv1alpha1.Condition{ - {Type: druidv1alpha1.ConditionTypeAllMembersReady, Status: druidv1alpha1.ConditionTrue}, - }, - } - - return eb -} - -func (eb *EtcdBuilder) WithStorageProvider(provider druidv1alpha1.StorageProvider) *EtcdBuilder { - // TODO: there is no default case right now which is not very right, returning an error in a default case makes it difficult to chain - // This should be improved later - switch provider { - case "aws": - return eb.WithProviderS3() - case "azure": - return eb.WithProviderABS() - case "alicloud": - return eb.WithProviderOSS() - case "gcp": - return eb.WithProviderGCS() - case "openstack": - return eb.WithProviderSwift() - case "local": - return eb.WithProviderLocal() - default: - return eb - } -} - -func (eb *EtcdBuilder) WithProviderS3() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "aws", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderABS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "azure", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderGCS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "gcp", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderSwift() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "openstack", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderOSS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "alicloud", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderLocal() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStoreForLocal( - eb.etcd.Name, - ) - return eb -} - -func (eb *EtcdBuilder) WithEtcdClientPort(clientPort *int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Etcd.ClientPort = clientPort - return eb -} - -func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - - if eb.etcd.Spec.Etcd.ClientService == nil { - eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} - } - - eb.etcd.Spec.Etcd.ClientService.Labels = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) - return eb -} - -func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]string) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - - if eb.etcd.Spec.Etcd.ClientService == nil { - eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} - } - - eb.etcd.Spec.Etcd.ClientService.Annotations = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) - return eb -} - -func (eb *EtcdBuilder) WithEtcdServerPort(serverPort *int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Etcd.ServerPort = serverPort - return eb -} - -func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Port = port - return eb -} - -func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { - return eb.etcd -} - -func getEtcdWithoutDefaults(name, namespace string) *druidv1alpha1.Etcd { - return &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: druidv1alpha1.EtcdSpec{ - Annotations: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - "role": "test", - }, - Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - }, - Replicas: 1, - Backup: druidv1alpha1.BackupSpec{}, - Etcd: druidv1alpha1.EtcdConfig{}, - Common: druidv1alpha1.SharedConfig{}, - }, - } -} - -func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { - return &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: types.UID(uid), - }, - Spec: druidv1alpha1.EtcdSpec{ - Annotations: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - "role": "test", - }, - Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - }, - Replicas: 1, - StorageCapacity: &storageCapacity, - StorageClass: &storageClass, - PriorityClassName: &priorityClassName, - VolumeClaimTemplate: &volumeClaimTemplateName, - Backup: druidv1alpha1.BackupSpec{ - Image: &imageBR, - Port: &backupPort, - FullSnapshotSchedule: &snapshotSchedule, - GarbageCollectionPolicy: &garbageCollectionPolicy, - GarbageCollectionPeriod: &garbageCollectionPeriod, - DeltaSnapshotPeriod: &deltaSnapshotPeriod, - DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, - EtcdSnapshotTimeout: &etcdSnapshotTimeout, - - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": ParseQuantity("500m"), - "memory": ParseQuantity("2Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": ParseQuantity("23m"), - "memory": ParseQuantity("128Mi"), - }, - }, - Store: &druidv1alpha1.StoreSpec{ - SecretRef: &corev1.SecretReference{ - Name: "etcd-backup", - }, - Container: &container, - Provider: &localProvider, - Prefix: prefix, - }, - }, - Etcd: druidv1alpha1.EtcdConfig{ - Quota: "a, - Metrics: &metricsBasic, - Image: &imageEtcd, - DefragmentationSchedule: &defragSchedule, - EtcdDefragTimeout: &etcdDefragTimeout, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": ParseQuantity("2500m"), - "memory": ParseQuantity("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": ParseQuantity("500m"), - "memory": ParseQuantity("1000Mi"), - }, - }, - ClientPort: &clientPort, - ServerPort: &serverPort, - }, - Common: druidv1alpha1.SharedConfig{ - AutoCompactionMode: &autoCompactionMode, - AutoCompactionRetention: &autoCompactionRetention, - }, - }, - } -} - -func getBackupStore(name string, provider druidv1alpha1.StorageProvider) *druidv1alpha1.StoreSpec { - return &druidv1alpha1.StoreSpec{ - Container: &container, - Prefix: name, - Provider: &provider, - SecretRef: &corev1.SecretReference{ - Name: "etcd-backup", - }, - } -} - -func getBackupStoreForLocal(name string) *druidv1alpha1.StoreSpec { - provider := druidv1alpha1.StorageProvider("local") - return &druidv1alpha1.StoreSpec{ - Container: &container, - Prefix: name, - Provider: &provider, - } -} - func CheckEtcdOwnerReference(refs []metav1.OwnerReference, etcd *druidv1alpha1.Etcd) bool { for _, ownerRef := range refs { if ownerRef.UID == etcd.UID { diff --git a/test/utils/misc.go b/test/utils/misc.go index b45ea2898..4927ad97f 100644 --- a/test/utils/misc.go +++ b/test/utils/misc.go @@ -6,10 +6,11 @@ package utils import ( . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" + + "os" + gomegatypes "github.com/onsi/gomega/types" "k8s.io/apimachinery/pkg/api/resource" - "os" ) func MatchFinalizer(finalizer string) gomegatypes.GomegaMatcher { From bda2cda8e91fdd73d5711049ff5775a98a4ee445 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 11:53:20 +0530 Subject: [PATCH 050/235] Removed unused file --- internal/operator/statefulset/helper.go | 870 ------------------------ 1 file changed, 870 deletions(-) delete mode 100644 internal/operator/statefulset/helper.go diff --git a/internal/operator/statefulset/helper.go b/internal/operator/statefulset/helper.go deleted file mode 100644 index 6b6772884..000000000 --- a/internal/operator/statefulset/helper.go +++ /dev/null @@ -1,870 +0,0 @@ -package statefulset - -//import ( -// "encoding/json" -// "fmt" -// "strconv" -// "strings" -// -// druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -// "github.com/gardener/etcd-druid/internal/operator/resource" -// "github.com/gardener/etcd-druid/internal/utils" -// gardenerutils "github.com/gardener/gardener/pkg/utils" -// "github.com/go-logr/logr" -// appsv1 "k8s.io/api/apps/v1" -// corev1 "k8s.io/api/core/v1" -// apiresource "k8s.io/apimachinery/pkg/api/resource" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/util/intstr" -// "k8s.io/utils/pointer" -// "sigs.k8s.io/controller-runtime/pkg/client" -//) -// -//type consistencyLevel string -// -//const ( -// linearizable consistencyLevel = "linearizable" -// serializable consistencyLevel = "serializable" -// defaultBackupPort int32 = 8080 -// defaultServerPort int32 = 2380 -// defaultClientPort int32 = 2379 -// defaultWrapperPort int32 = 9095 -// defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi -// defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi -// defaultHeartbeatDuration = "10s" -// defaultGbcPolicy = "LimitBased" -// defaultAutoCompactionRetention = "30m" -// defaultEtcdSnapshotTimeout = "15m" -// defaultEtcdDefragTimeout = "15m" -// defaultAutoCompactionMode = "periodic" -// defaultEtcdConnectionTimeout = "5m" -//) -// -//var defaultStorageCapacity = apiresource.MustParse("16Gi") -//var defaultResourceRequirements = corev1.ResourceRequirements{ -// Requests: corev1.ResourceList{ -// corev1.ResourceCPU: apiresource.MustParse("50m"), -// corev1.ResourceMemory: apiresource.MustParse("128Mi"), -// }, -//} -// -//func extractObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd) metav1.ObjectMeta { -// return metav1.ObjectMeta{ -// Name: etcd.Name, -// Namespace: etcd.Namespace, -// Labels: etcd.GetDefaultLabels(), -// OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, -// } -//} -//func extractPodObjectMetaFromEtcd(etcd *druidv1alpha1.Etcd, configMapChecksum string) metav1.ObjectMeta { -// return metav1.ObjectMeta{ -// Labels: utils.MergeStringMaps(make(map[string]string), etcd.Spec.Labels, etcd.GetDefaultLabels()), -// Annotations: utils.MergeStringMaps(map[string]string{ -// resource.ConfigMapCheckSumKey: configMapChecksum, -// }, etcd.Spec.Annotations), -// } -//} -// -//func getEtcdCommandArgs(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []string { -// if !useEtcdWrapper { -// // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh -// return []string{} -// } -// //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA -// command := []string{"" + "start-etcd"} -// command = append(command, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", etcd.Name)) -// command = append(command, fmt.Sprintf("--etcd-server-name=%s-local", etcd.Name)) -// -// clientURLTLS := etcd.Spec.Etcd.ClientUrlTLS -// if clientURLTLS == nil { -// command = append(command, "--backup-restore-tls-enabled=false") -// } else { -// dataKey := "ca.crt" -// if clientURLTLS.TLSCASecretRef.DataKey != nil { -// dataKey = *clientURLTLS.TLSCASecretRef.DataKey -// } -// command = append(command, "--backup-restore-tls-enabled=true") -// command = append(command, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") -// command = append(command, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") -// command = append(command, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) -// } -// -// return command -//} -// -//func getReadinessHandler(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { -// if etcd.Spec.Replicas > 1 { -// // TODO(timuthy): Special handling for multi-node etcd can be removed as soon as -// // etcd-backup-restore supports `/healthz` for etcd followers, see https://github.com/gardener/etcd-backup-restore/pull/491. -// return getReadinessHandlerForMultiNode(useEtcdWrapper, etcd) -// } -// return getReadinessHandlerForSingleNode(etcd) -//} -// -//func getReadinessHandlerForSingleNode(etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { -// scheme := corev1.URISchemeHTTPS -// if etcd.Spec.Backup.TLS == nil { -// scheme = corev1.URISchemeHTTP -// } -// -// return corev1.ProbeHandler{ -// HTTPGet: &corev1.HTTPGetAction{ -// Path: "/healthz", -// Port: intstr.FromInt(int(defaultBackupPort)), -// Scheme: scheme, -// }, -// } -//} -// -//func getReadinessHandlerForMultiNode(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) corev1.ProbeHandler { -// if useEtcdWrapper { -// //TODO @aaronfern: remove this feature gate when useEtcdWrapper becomes GA -// scheme := corev1.URISchemeHTTPS -// if etcd.Spec.Backup.TLS == nil { -// scheme = corev1.URISchemeHTTP -// } -// -// return corev1.ProbeHandler{ -// HTTPGet: &corev1.HTTPGetAction{ -// Path: "/readyz", -// Port: intstr.FromInt(int(defaultWrapperPort)), -// Scheme: scheme, -// }, -// } -// } -// -// return corev1.ProbeHandler{ -// Exec: &corev1.ExecAction{ -// Command: getProbeCommand(etcd, linearizable), -// }, -// } -//} -// -//func getEtcdPorts() []corev1.ContainerPort { -// return []corev1.ContainerPort{ -// { -// Name: "server", -// Protocol: "TCP", -// ContainerPort: defaultServerPort, -// }, -// { -// Name: "client", -// Protocol: "TCP", -// ContainerPort: defaultClientPort, -// }, -// } -//} -// -//func getProbeCommand(etcd *druidv1alpha1.Etcd, consistency consistencyLevel) []string { -// var etcdCtlCommand strings.Builder -// -// etcdCtlCommand.WriteString("ETCDCTL_API=3 etcdctl") -// -// if etcd.Spec.Etcd.ClientUrlTLS != nil { -// dataKey := "ca.crt" -// if etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey != nil { -// dataKey = *etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey -// } -// -// etcdCtlCommand.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) -// etcdCtlCommand.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") -// etcdCtlCommand.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") -// etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) -// -// } else { -// etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) -// } -// -// etcdCtlCommand.WriteString(" get foo") -// -// switch consistency { -// case linearizable: -// etcdCtlCommand.WriteString(" --consistency=l") -// case serializable: -// etcdCtlCommand.WriteString(" --consistency=s") -// } -// -// return []string{ -// "/bin/sh", -// "-ec", -// etcdCtlCommand.String(), -// } -//} -// -//func getEtcdResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { -// if etcd.Spec.Etcd.Resources != nil { -// return *etcd.Spec.Etcd.Resources -// } -// -// return defaultResourceRequirements -//} -// -//func getEtcdEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { -// protocol := "http" -// if etcd.Spec.Backup.TLS != nil { -// protocol = "https" -// } -// -// endpoint := fmt.Sprintf("%s://%s-local:%d", protocol, etcd.Name, defaultBackupPort) -// -// return []corev1.EnvVar{ -// getEnvVarFromValue("ENABLE_TLS", strconv.FormatBool(etcd.Spec.Backup.TLS != nil)), -// getEnvVarFromValue("BACKUP_ENDPOINT", endpoint), -// } -//} -// -//func getEnvVarFromValue(name, value string) corev1.EnvVar { -// return corev1.EnvVar{ -// Name: name, -// Value: value, -// } -//} -// -//func getEnvVarFromField(name, fieldPath string) corev1.EnvVar { -// return corev1.EnvVar{ -// Name: name, -// ValueFrom: &corev1.EnvVarSource{ -// FieldRef: &corev1.ObjectFieldSelector{ -// FieldPath: fieldPath, -// }, -// }, -// } -//} -// -//func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { -// return corev1.EnvVar{ -// Name: name, -// ValueFrom: &corev1.EnvVarSource{ -// SecretKeyRef: &corev1.SecretKeySelector{ -// LocalObjectReference: corev1.LocalObjectReference{ -// Name: secretName, -// }, -// Key: secretKey, -// }, -// }, -// } -//} -// -//func getEtcdVolumeMounts(etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { -// volumeClaimTemplateName := etcd.Name -// if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { -// volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate -// } -// -// vms := []corev1.VolumeMount{ -// { -// Name: volumeClaimTemplateName, -// MountPath: "/var/etcd/data/", -// }, -// } -// -// vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) -// -// return vms -//} -// -//func getSecretVolumeMounts(clientUrlTLS, peerUrlTLS *druidv1alpha1.TLSConfig) []corev1.VolumeMount { -// var vms []corev1.VolumeMount -// -// if clientUrlTLS != nil { -// vms = append(vms, corev1.VolumeMount{ -// Name: "client-url-ca-etcd", -// MountPath: "/var/etcd/ssl/client/ca", -// }, corev1.VolumeMount{ -// Name: "client-url-etcd-server-tls", -// MountPath: "/var/etcd/ssl/client/server", -// }, corev1.VolumeMount{ -// Name: "client-url-etcd-client-tls", -// MountPath: "/var/etcd/ssl/client/client", -// }) -// } -// -// if peerUrlTLS != nil { -// vms = append(vms, corev1.VolumeMount{ -// Name: "peer-url-ca-etcd", -// MountPath: "/var/etcd/ssl/peer/ca", -// }, corev1.VolumeMount{ -// Name: "peer-url-etcd-server-tls", -// MountPath: "/var/etcd/ssl/peer/server", -// }) -// } -// -// return vms -//} -// -//func getBackupRestoreCommandArgs(etcd *druidv1alpha1.Etcd) []string { -// command := []string{"server"} -// -// if etcd.Spec.Backup.Store != nil { -// command = append(command, "--enable-snapshot-lease-renewal=true") -// command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) -// command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) -// } -// -// if etcd.Spec.Etcd.DefragmentationSchedule != nil { -// command = append(command, "--defragmentation-schedule="+*etcd.Spec.Etcd.DefragmentationSchedule) -// } -// -// if etcd.Spec.Backup.FullSnapshotSchedule != nil { -// command = append(command, "--schedule="+*etcd.Spec.Backup.FullSnapshotSchedule) -// } -// -// garbageCollectionPolicy := defaultGbcPolicy -// if etcd.Spec.Backup.GarbageCollectionPolicy != nil { -// garbageCollectionPolicy = string(*etcd.Spec.Backup.GarbageCollectionPolicy) -// } -// -// command = append(command, "--garbage-collection-policy="+garbageCollectionPolicy) -// if garbageCollectionPolicy == "LimitBased" { -// command = append(command, "--max-backups=7") -// } -// -// command = append(command, "--data-dir=/var/etcd/data/new.etcd") -// command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") -// -// if etcd.Spec.Backup.Store != nil { -// store, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) -// if err != nil { -// return nil -// } -// command = append(command, "--storage-provider="+store) -// command = append(command, "--store-prefix="+string(etcd.Spec.Backup.Store.Prefix)) -// } -// -// var quota = defaultQuota -// if etcd.Spec.Etcd.Quota != nil { -// quota = etcd.Spec.Etcd.Quota.Value() -// } -// -// command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) -// -// if pointer.BoolDeref(etcd.Spec.Backup.EnableProfiling, false) { -// command = append(command, "--enable-profiling=true") -// } -// -// if etcd.Spec.Etcd.ClientUrlTLS != nil { -// command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") -// command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") -// command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+pointer.StringDeref(etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt")) -// command = append(command, "--insecure-transport=false") -// command = append(command, "--insecure-skip-tls-verify=false") -// command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", etcd.Name, defaultClientPort)) -// command = append(command, fmt.Sprintf("--service-endpoints=https://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) -// } else { -// command = append(command, "--insecure-transport=true") -// command = append(command, "--insecure-skip-tls-verify=true") -// command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", etcd.Name, defaultClientPort)) -// command = append(command, fmt.Sprintf("--service-endpoints=http://%s:%d", etcd.GetClientServiceName(), defaultClientPort)) -// -// } -// -// if etcd.Spec.Backup.TLS != nil { -// command = append(command, "--server-cert=/var/etcd/ssl/client/server/tls.crt") -// command = append(command, "--server-key=/var/etcd/ssl/client/server/tls.key") -// } -// -// command = append(command, "--etcd-connection-timeout="+defaultEtcdConnectionTimeout) -// -// if etcd.Spec.Backup.DeltaSnapshotPeriod != nil { -// command = append(command, "--delta-snapshot-period="+etcd.Spec.Backup.DeltaSnapshotPeriod.Duration.String()) -// } -// -// if etcd.Spec.Backup.DeltaSnapshotRetentionPeriod != nil { -// command = append(command, "--delta-snapshot-retention-period="+etcd.Spec.Backup.DeltaSnapshotRetentionPeriod.Duration.String()) -// } -// -// var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit -// if etcd.Spec.Backup.DeltaSnapshotMemoryLimit != nil { -// deltaSnapshotMemoryLimit = etcd.Spec.Backup.DeltaSnapshotMemoryLimit.Value() -// } -// -// command = append(command, "--delta-snapshot-memory-limit="+fmt.Sprint(deltaSnapshotMemoryLimit)) -// -// if etcd.Spec.Backup.GarbageCollectionPeriod != nil { -// command = append(command, "--garbage-collection-period="+etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String()) -// } -// -// if etcd.Spec.Backup.SnapshotCompression != nil { -// if pointer.BoolDeref(etcd.Spec.Backup.SnapshotCompression.Enabled, false) { -// command = append(command, "--compress-snapshots="+fmt.Sprint(*etcd.Spec.Backup.SnapshotCompression.Enabled)) -// } -// if etcd.Spec.Backup.SnapshotCompression.Policy != nil { -// command = append(command, "--compression-policy="+string(*etcd.Spec.Backup.SnapshotCompression.Policy)) -// } -// } -// -// compactionMode := defaultAutoCompactionMode -// if etcd.Spec.Common.AutoCompactionMode != nil { -// compactionMode = string(*etcd.Spec.Common.AutoCompactionMode) -// } -// command = append(command, "--auto-compaction-mode="+compactionMode) -// -// compactionRetention := defaultAutoCompactionRetention -// if etcd.Spec.Common.AutoCompactionRetention != nil { -// compactionRetention = *etcd.Spec.Common.AutoCompactionRetention -// } -// command = append(command, "--auto-compaction-retention="+compactionRetention) -// -// etcdSnapshotTimeout := defaultEtcdSnapshotTimeout -// if etcd.Spec.Backup.EtcdSnapshotTimeout != nil { -// etcdSnapshotTimeout = etcd.Spec.Backup.EtcdSnapshotTimeout.Duration.String() -// } -// command = append(command, "--etcd-snapshot-timeout="+etcdSnapshotTimeout) -// -// etcdDefragTimeout := defaultEtcdDefragTimeout -// if etcd.Spec.Etcd.EtcdDefragTimeout != nil { -// etcdDefragTimeout = etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String() -// } -// command = append(command, "--etcd-defrag-timeout="+etcdDefragTimeout) -// -// command = append(command, "--snapstore-temp-directory=/var/etcd/data/temp") -// command = append(command, "--enable-member-lease-renewal=true") -// -// heartbeatDuration := defaultHeartbeatDuration -// if etcd.Spec.Etcd.HeartbeatDuration != nil { -// heartbeatDuration = etcd.Spec.Etcd.HeartbeatDuration.Duration.String() -// } -// command = append(command, "--k8s-heartbeat-duration="+heartbeatDuration) -// -// if etcd.Spec.Backup.LeaderElection != nil { -// if etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout != nil { -// command = append(command, "--etcd-connection-timeout-leader-election="+etcd.Spec.Backup.LeaderElection.EtcdConnectionTimeout.Duration.String()) -// } -// -// if etcd.Spec.Backup.LeaderElection.ReelectionPeriod != nil { -// command = append(command, "--reelection-period="+etcd.Spec.Backup.LeaderElection.ReelectionPeriod.Duration.String()) -// } -// } -// -// return command -//} -// -//func getBackupPorts() []corev1.ContainerPort { -// return []corev1.ContainerPort{ -// { -// Name: "server", -// Protocol: "TCP", -// ContainerPort: defaultBackupPort, -// }, -// } -//} -// -//func getBackupResources(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { -// if etcd.Spec.Backup.Resources != nil { -// return *etcd.Spec.Backup.Resources -// } -// return defaultResourceRequirements -//} -// -//func getBackupRestoreEnvVars(etcd *druidv1alpha1.Etcd) []corev1.EnvVar { -// var ( -// env []corev1.EnvVar -// storageContainer string -// storeValues = etcd.Spec.Backup.Store -// ) -// -// if etcd.Spec.Backup.Store != nil { -// storageContainer = pointer.StringDeref(etcd.Spec.Backup.Store.Container, "") -// } -// -// // TODO(timuthy, shreyas-s-rao): Move STORAGE_CONTAINER a few lines below so that we can append and exit in one step. This should only be done in a release where a restart of etcd is acceptable. -// env = append(env, getEnvVarFromValue("STORAGE_CONTAINER", storageContainer)) -// env = append(env, getEnvVarFromField("POD_NAME", "metadata.name")) -// env = append(env, getEnvVarFromField("POD_NAMESPACE", "metadata.namespace")) -// -// if storeValues == nil { -// return env -// } -// -// provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) -// if err != nil { -// return env -// } -// -// // TODO(timuthy): move this to a non root path when we switch to a rootless distribution -// const credentialsMountPath = "/var/etcd-backup" -// switch provider { -// case utils.S3: -// env = append(env, getEnvVarFromValue("AWS_APPLICATION_CREDENTIALS", credentialsMountPath)) -// -// case utils.ABS: -// env = append(env, getEnvVarFromValue("AZURE_APPLICATION_CREDENTIALS", credentialsMountPath)) -// -// case utils.GCS: -// env = append(env, getEnvVarFromValue("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) -// -// case utils.Swift: -// env = append(env, getEnvVarFromValue("OPENSTACK_APPLICATION_CREDENTIALS", credentialsMountPath)) -// -// case utils.OSS: -// env = append(env, getEnvVarFromValue("ALICLOUD_APPLICATION_CREDENTIALS", credentialsMountPath)) -// -// case utils.ECS: -// env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) -// env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) -// env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) -// -// case utils.OCS: -// env = append(env, getEnvVarFromValue("OPENSHIFT_APPLICATION_CREDENTIALS", credentialsMountPath)) -// } -// -// return env -//} -// -//func getBackupRestoreVolumeMounts(useEtcdWrapper bool, etcd *druidv1alpha1.Etcd) []corev1.VolumeMount { -// volumeClaimTemplateName := etcd.Name -// if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { -// volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate -// } -// vms := []corev1.VolumeMount{ -// { -// Name: volumeClaimTemplateName, -// MountPath: "/var/etcd/data", -// }, -// { -// Name: "etcd-config-file", -// MountPath: "/var/etcd/config/", -// }, -// } -// -// vms = append(vms, getSecretVolumeMounts(etcd.Spec.Etcd.ClientUrlTLS, etcd.Spec.Etcd.PeerUrlTLS)...) -// -// if etcd.Spec.Backup.Store == nil { -// return vms -// } -// -// provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) -// if err != nil { -// return vms -// } -// -// switch provider { -// case utils.Local: -// if etcd.Spec.Backup.Store.Container != nil { -// if useEtcdWrapper { -// vms = append(vms, corev1.VolumeMount{ -// Name: "host-storage", -// MountPath: "/home/nonroot/" + pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), -// }) -// } else { -// vms = append(vms, corev1.VolumeMount{ -// Name: "host-storage", -// MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), -// }) -// } -// } -// case utils.GCS: -// vms = append(vms, corev1.VolumeMount{ -// Name: "etcd-backup", -// MountPath: "/var/.gcp/", -// }) -// case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: -// vms = append(vms, corev1.VolumeMount{ -// Name: "etcd-backup", -// MountPath: "/var/etcd-backup/", -// }) -// } -// -// return vms -//} -// -//func getvolumeClaimTemplateName(etcd *druidv1alpha1.Etcd) string { -// volumeClaimTemplateName := etcd.Name -// if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { -// volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate -// } -// return volumeClaimTemplateName -//} -// -//func getStorageReq(etcd *druidv1alpha1.Etcd) corev1.ResourceRequirements { -// storageCapacity := defaultStorageCapacity -// if etcd.Spec.StorageCapacity != nil { -// storageCapacity = *etcd.Spec.StorageCapacity -// } -// -// return corev1.ResourceRequirements{ -// Requests: corev1.ResourceList{ -// corev1.ResourceStorage: storageCapacity, -// }, -// } -//} -// -//func addEtcdContainer(etcdImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { -// return &corev1.Container{ -// Name: "etcd", -// Image: *etcdImage, -// ImagePullPolicy: corev1.PullIfNotPresent, -// Args: getEtcdCommandArgs(isEtcdWrapperEnabled, etcd), -// ReadinessProbe: &corev1.Probe{ -// ProbeHandler: getReadinessHandler(isEtcdWrapperEnabled, etcd), -// InitialDelaySeconds: 15, -// PeriodSeconds: 5, -// FailureThreshold: 5, -// }, -// Ports: getEtcdPorts(), -// Resources: getEtcdResources(etcd), -// Env: getEtcdEnvVars(etcd), -// VolumeMounts: getEtcdVolumeMounts(etcd), -// } -//} -// -//func addBackupRestoreContainer(etcdBackupImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) *corev1.Container { -// return &corev1.Container{ -// Name: "backup-restore", -// Image: *etcdBackupImage, -// ImagePullPolicy: corev1.PullIfNotPresent, -// Args: getBackupRestoreCommandArgs(etcd), -// Ports: getBackupPorts(), -// Resources: getBackupResources(etcd), -// Env: getBackupRestoreEnvVars(etcd), -// VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), -// SecurityContext: &corev1.SecurityContext{ -// Capabilities: &corev1.Capabilities{ -// Add: []corev1.Capability{ -// "SYS_PTRACE", -// }, -// }, -// }, -// } -//} -// -//func addInitContainersIfWrapperEnabled(initContainerImage *string, isEtcdWrapperEnabled bool, etcd *druidv1alpha1.Etcd) []corev1.Container { -// if !isEtcdWrapperEnabled { -// return []corev1.Container{} -// } -// -// // Initialize the slice with the 'change-permissions' container -// initContainers := []corev1.Container{ -// { -// Name: "change-permissions", -// Image: *initContainerImage, -// ImagePullPolicy: corev1.PullIfNotPresent, -// Command: []string{"sh", "-c", "--"}, -// Args: []string{"chown -R 65532:65532 /var/etcd/data"}, -// VolumeMounts: getEtcdVolumeMounts(etcd), -// SecurityContext: &corev1.SecurityContext{ -// RunAsGroup: pointer.Int64(0), -// RunAsNonRoot: pointer.Bool(false), -// RunAsUser: pointer.Int64(0), -// }, -// }, -// } -// -// if etcd.Spec.Backup.Store != nil { -// prov, _ := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) -// if prov == utils.Local { -// initContainers = append(initContainers, corev1.Container{ -// Name: "change-backup-bucket-permissions", -// Image: *initContainerImage, -// ImagePullPolicy: corev1.PullIfNotPresent, -// Command: []string{"sh", "-c", "--"}, -// Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *etcd.Spec.Backup.Store.Container)}, -// VolumeMounts: getBackupRestoreVolumeMounts(isEtcdWrapperEnabled, etcd), -// SecurityContext: &corev1.SecurityContext{ -// RunAsGroup: pointer.Int64(0), -// RunAsNonRoot: pointer.Bool(false), -// RunAsUser: pointer.Int64(0), -// }, -// }) -// } -// } -// -// return initContainers -//} -// -//func getVolumeClaimTemplates(etcd *druidv1alpha1.Etcd) []corev1.PersistentVolumeClaim { -// return []corev1.PersistentVolumeClaim{{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: getvolumeClaimTemplateName(etcd), -// }, -// Spec: corev1.PersistentVolumeClaimSpec{ -// AccessModes: []corev1.PersistentVolumeAccessMode{ -// corev1.ReadWriteOnce, -// }, -// Resources: getStorageReq(etcd), -// StorageClassName: etcd.Spec.StorageClass, -// }, -// }, -// } -//} -// -//func addSecurityContextIfWrapperEnabled(isEtcdWrapperEnabled bool) *corev1.PodSecurityContext { -// return &corev1.PodSecurityContext{ -// RunAsGroup: pointer.Int64(65532), -// RunAsNonRoot: pointer.Bool(true), -// RunAsUser: pointer.Int64(65532), -// } -//} -// -//func getPriorityClassName(etcd *druidv1alpha1.Etcd) string { -// if etcd.Spec.PriorityClassName != nil { -// return *etcd.Spec.PriorityClassName -// } -// return "" -//} -// -//// hasImmutableFieldChanged checks if any immutable fields have changed in the StatefulSet -//// specification compared to the Etcd object. It returns true if there are changes in the immutable fields. -//// Currently, it checks for changes in the ServiceName and PodManagementPolicy. -//func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { -// return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement -//} -// -//func getVolumes(client client.Client, logger logr.Logger, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]corev1.Volume, error) { -// vs := []corev1.Volume{ -// { -// Name: "etcd-config-file", -// VolumeSource: corev1.VolumeSource{ -// ConfigMap: &corev1.ConfigMapVolumeSource{ -// LocalObjectReference: corev1.LocalObjectReference{ -// Name: etcd.GetConfigMapName(), -// }, -// Items: []corev1.KeyToPath{ -// { -// Key: "etcd.conf.yaml", -// Path: "etcd.conf.yaml", -// }, -// }, -// DefaultMode: pointer.Int32(0644), -// }, -// }, -// }, -// } -// -// if etcd.Spec.Etcd.ClientUrlTLS != nil { -// vs = append(vs, corev1.Volume{ -// Name: "client-url-ca-etcd", -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, -// }, -// }, -// }, -// corev1.Volume{ -// Name: "client-url-etcd-server-tls", -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, -// }, -// }, -// }, -// corev1.Volume{ -// Name: "client-url-etcd-client-tls", -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, -// }, -// }, -// }) -// } -// -// if etcd.Spec.Etcd.PeerUrlTLS != nil { -// vs = append(vs, corev1.Volume{ -// Name: "peer-url-ca-etcd", -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, -// }, -// }, -// }, -// corev1.Volume{ -// Name: "peer-url-etcd-server-tls", -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, -// }, -// }, -// }) -// } -// -// if etcd.Spec.Backup.Store == nil { -// return vs, nil -// } -// -// storeValues := etcd.Spec.Backup.Store -// provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) -// if err != nil { -// return vs, nil -// } -// -// switch provider { -// case "Local": -// hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, client, logger, storeValues, etcd.GetNamespace()) -// if err != nil { -// return nil, err -// } -// -// hpt := corev1.HostPathDirectory -// vs = append(vs, corev1.Volume{ -// Name: "host-storage", -// VolumeSource: corev1.VolumeSource{ -// HostPath: &corev1.HostPathVolumeSource{ -// Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), -// Type: &hpt, -// }, -// }, -// }) -// case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: -// if storeValues.SecretRef == nil { -// return nil, fmt.Errorf("no secretRef configured for backup store") -// } -// -// vs = append(vs, corev1.Volume{ -// Name: "etcd-backup", -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: storeValues.SecretRef.Name, -// }, -// }, -// }) -// } -// -// return vs, nil -//} -// -//func getConfigMapChecksum(cl client.Client, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (string, error) { -// cm := &corev1.ConfigMap{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: etcd.GetConfigMapName(), -// Namespace: etcd.Namespace, -// }, -// } -// if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { -// -// return "", err -// } -// jsonString, err := json.Marshal(cm.Data) -// if err != nil { -// return "", err -// } -// -// return gardenerutils.ComputeSHA256Hex(jsonString), nil -//} -// -//func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, logger logr.Logger, opName string, sts *appsv1.StatefulSet) error { -// // Get all Pods belonging to the StatefulSet -// podList := &corev1.PodList{} -// listOpts := []client.ListOption{ -// client.InNamespace(sts.Namespace), -// client.MatchingLabels(sts.Spec.Selector.MatchLabels), -// } -// -// if err := cl.List(ctx, podList, listOpts...); err != nil { -// logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) -// return err -// } -// -// for _, pod := range podList.Items { -// if err := cl.Delete(ctx, &pod); err != nil { -// logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) -// return err -// } -// } -// -// return nil -//} -// -//// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled -//func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { -// return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil -//} From 1eb4d23dcaa417f234037e54a389515b6f30e77b Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 5 Jan 2024 11:59:10 +0530 Subject: [PATCH 051/235] minor improvement in test utility fn and removal of dead code --- internal/operator/statefulset/statefulset.go | 70 +------------------ .../controllers/compaction/reconciler_test.go | 2 +- test/utils/etcd.go | 5 +- 3 files changed, 6 insertions(+), 71 deletions(-) diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 2a4068185..477bdfb82 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -42,19 +42,15 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) if err != nil { return fmt.Errorf("error getting existing StatefulSet: %w", err) } - if existingSts == nil { return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) } - - if err := r.handlePeerTLSEnabled(ctx, etcd, existingSts); err != nil { + if err = r.handlePeerTLSEnabled(ctx, etcd, existingSts); err != nil { return fmt.Errorf("error handling peer TLS: %w", err) } - - if err := r.handleImmutableFieldUpdates(ctx, etcd, existingSts); err != nil { + if err = r.handleImmutableFieldUpdates(ctx, etcd, existingSts); err != nil { return fmt.Errorf("error handling immutable field updates: %w", err) } - return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) } @@ -91,68 +87,6 @@ func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alph return nil } -//func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { -// desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) -// etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.useEtcdWrapper) -// if err != nil { -// return err -// } -// podVolumes, err := getVolumes(r.client, r.logger, ctx, etcd) -// if err != nil { -// return err -// } -// -// configMapChecksum, err := getConfigMapChecksum(r.client, ctx, etcd) -// if err != nil { -// return err -// } -// -// mutatingFn := func() error { -// desiredStatefulSet.ObjectMeta = extractObjectMetaFromEtcd(etcd) -// desiredStatefulSet.Spec = appsv1.StatefulSetSpec{ -// Replicas: &replicas, -// ServiceName: etcd.GetPeerServiceName(), -// Selector: &metav1.LabelSelector{ -// MatchLabels: etcd.GetDefaultLabels(), -// }, -// PodManagementPolicy: appsv1.ParallelPodManagement, -// UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ -// Type: appsv1.RollingUpdateStatefulSetStrategyType, -// }, -// VolumeClaimTemplates: getVolumeClaimTemplates(etcd), -// Template: corev1.PodTemplateSpec{ -// ObjectMeta: extractPodObjectMetaFromEtcd(etcd, configMapChecksum), -// Spec: corev1.PodSpec{ -// HostAliases: []corev1.HostAlias{{ -// IP: "127.0.0.1", -// Hostnames: []string{etcd.Name + "-local"}}}, -// ServiceAccountName: etcd.GetServiceAccountName(), -// Affinity: etcd.Spec.SchedulingConstraints.Affinity, -// TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, -// Containers: []corev1.Container{ -// *addEtcdContainer(etcdImage, r.useEtcdWrapper, etcd), -// *addBackupRestoreContainer(etcdBackupImage, r.useEtcdWrapper, etcd), -// }, -// InitContainers: addInitContainersIfWrapperEnabled(initContainerImage, r.useEtcdWrapper, etcd), -// ShareProcessNamespace: pointer.Bool(true), -// Volumes: podVolumes, -// SecurityContext: addSecurityContextIfWrapperEnabled(r.useEtcdWrapper), -// PriorityClassName: getPriorityClassName(etcd), -// }, -// }, -// } -// return nil -// } -// -// opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) -// if err != nil { -// return err -// } -// -// r.logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) -// return nil -//} - func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { if existingSts.Generation > 1 && hasImmutableFieldChanged(existingSts, etcd) { ctx.Logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index d9842a0ee..d513adb72 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -704,7 +704,7 @@ func etcdSnapshotLeaseIsCorrectlyReconciled(c client.Client, instance *druidv1al return err } - if !testutils.CheckEtcdOwnerReference(lease.GetOwnerReferences(), instance) { + if !testutils.CheckEtcdOwnerReference(lease.GetOwnerReferences(), instance.UID) { return fmt.Errorf("ownerReference does not exists for lease") } return nil diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 061affcfe..45d3b5b30 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -17,9 +17,10 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func CheckEtcdOwnerReference(refs []metav1.OwnerReference, etcd *druidv1alpha1.Etcd) bool { +// CheckEtcdOwnerReference checks if one of the owner references points to etcd owner UID. +func CheckEtcdOwnerReference(refs []metav1.OwnerReference, etcdOwnerUID types.UID) bool { for _, ownerRef := range refs { - if ownerRef.UID == etcd.UID { + if ownerRef.UID == etcdOwnerUID { return true } } From 30fdfa878651fa32be62b2af82ad573b9617ab14 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 5 Jan 2024 12:04:24 +0530 Subject: [PATCH 052/235] adapted sample package --- .../clientservice/clientservice_test.go | 27 ++++++++++--------- internal/operator/role/role_test.go | 1 + 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 internal/operator/role/role_test.go diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 53760dab2..484a9f02f 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -26,6 +26,7 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/etcd-druid/test/sample" + testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -46,7 +47,7 @@ const ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { internalErr := errors.New("test internal error") - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testcases := []struct { name string svcExists bool @@ -104,11 +105,11 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestClientServiceSync(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string svcExists bool - setupFn func(eb *testutils.EtcdBuilder) + setupFn func(eb *testsample.EtcdBuilder) expectError *druiderr.DruidError getErr *apierrors.StatusError }{ @@ -120,7 +121,7 @@ func TestClientServiceSync(t *testing.T) { name: "Update existing service", svcExists: true, - setupFn: func(eb *testutils.EtcdBuilder) { + setupFn: func(eb *testsample.EtcdBuilder) { eb.WithEtcdClientPort(nil). WithBackupPort(nil). WithEtcdServerPort(nil). @@ -134,11 +135,11 @@ func TestClientServiceSync(t *testing.T) { setupFn: nil, expectError: &druiderr.DruidError{ Code: ErrSyncingClientService, - Cause: fmt.Errorf("Fake get error"), + Cause: fmt.Errorf("fake get error"), Operation: "Sync", Message: "Error during create or update of client service", }, - getErr: apierrors.NewInternalError(errors.New("Fake get error")), + getErr: apierrors.NewInternalError(errors.New("fake get error")), }, } @@ -156,7 +157,7 @@ func TestClientServiceSync(t *testing.T) { fakeClientBuilder.WithGetError(tc.getErr) } if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewClientService(testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + fakeClientBuilder.WithObjects(sample.NewClientService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) } cl := fakeClientBuilder.Build() operator := New(cl) @@ -189,11 +190,11 @@ func TestClientServiceSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestClientServiceTriggerDelete(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string svcExists bool - setupFn func(eb *testutils.EtcdBuilder) + setupFn func(eb *testsample.EtcdBuilder) expectError *druiderr.DruidError deleteErr *apierrors.StatusError }{ @@ -204,7 +205,7 @@ func TestClientServiceTriggerDelete(t *testing.T) { { name: "Service Not Found - No Operation", svcExists: true, - setupFn: func(eb *testutils.EtcdBuilder) { + setupFn: func(eb *testsample.EtcdBuilder) { eb.WithEtcdClientPort(nil). WithBackupPort(nil). WithEtcdServerPort(nil). @@ -218,11 +219,11 @@ func TestClientServiceTriggerDelete(t *testing.T) { setupFn: nil, expectError: &druiderr.DruidError{ Code: ErrDeletingClientService, - Cause: errors.New("Fake delete error"), + Cause: errors.New("fake delete error"), Operation: "TriggerDelete", Message: "Failed to delete client service", }, - deleteErr: apierrors.NewInternalError(errors.New("Fake delete error")), + deleteErr: apierrors.NewInternalError(errors.New("fake delete error")), }, } g := NewWithT(t) @@ -239,7 +240,7 @@ func TestClientServiceTriggerDelete(t *testing.T) { fakeClientBuilder.WithDeleteError(tc.deleteErr) } if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewClientService(testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + fakeClientBuilder.WithObjects(sample.NewClientService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) } cl := fakeClientBuilder.Build() operator := New(cl) diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go new file mode 100644 index 000000000..e82991293 --- /dev/null +++ b/internal/operator/role/role_test.go @@ -0,0 +1 @@ +package role From 0fd816baa3c99693a332e9ef12479a7f14d14c4e Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 12:00:19 +0530 Subject: [PATCH 053/235] Reorder imports --- test/utils/misc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/utils/misc.go b/test/utils/misc.go index 4927ad97f..f1ff0ffb8 100644 --- a/test/utils/misc.go +++ b/test/utils/misc.go @@ -5,10 +5,10 @@ package utils import ( - . "github.com/onsi/gomega" - "os" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" gomegatypes "github.com/onsi/gomega/types" "k8s.io/apimachinery/pkg/api/resource" ) From 9890f255ccb6f26b17e5615bdf64659ef9261fec Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 12:10:38 +0530 Subject: [PATCH 054/235] Fix lint errors --- test/utils/configmap.go | 2 +- test/utils/service.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/utils/configmap.go b/test/utils/configmap.go index cd2a472d7..8ec22893c 100644 --- a/test/utils/configmap.go +++ b/test/utils/configmap.go @@ -28,7 +28,7 @@ func ConfigMapIsCorrectlyReconciled(c client.Client, timeout time.Duration, inst return err } - if !CheckEtcdOwnerReference(cm.GetOwnerReferences(), instance) { + if !CheckEtcdOwnerReference(cm.GetOwnerReferences(), instance.UID) { return fmt.Errorf("ownerReference does not exists") } return nil diff --git a/test/utils/service.go b/test/utils/service.go index 823dce9bf..a9ca0dacf 100644 --- a/test/utils/service.go +++ b/test/utils/service.go @@ -28,7 +28,7 @@ func ClientServiceIsCorrectlyReconciled(c client.Client, timeout time.Duration, return err } - if !CheckEtcdOwnerReference(svc.GetOwnerReferences(), instance) { + if !CheckEtcdOwnerReference(svc.GetOwnerReferences(), instance.UID) { return fmt.Errorf("ownerReference does not exists") } return nil @@ -46,7 +46,7 @@ func PeerServiceIsCorrectlyReconciled(c client.Client, timeout time.Duration, in return err } - if !CheckEtcdOwnerReference(svc.GetOwnerReferences(), instance) { + if !CheckEtcdOwnerReference(svc.GetOwnerReferences(), instance.UID) { return fmt.Errorf("ownerReference does not exists") } return nil From a18ecf12a7c0ed18678ae1647104df774fce8b11 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 5 Jan 2024 13:07:18 +0530 Subject: [PATCH 055/235] changed the error code constants, added sample for role --- .../operator/clientservice/clientservice.go | 12 +++--- .../clientservice/clientservice_test.go | 10 ++--- internal/operator/role/role.go | 30 ++++++++++++-- test/sample/role.go | 39 +++++++++++++++++++ 4 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 test/sample/role.go diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 461b4f311..69344cd24 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -1,6 +1,8 @@ package clientservice import ( + "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" @@ -21,8 +23,8 @@ const ( ) const ( - ErrDeletingClientService druidv1alpha1.ErrorCode = "ERR_DELETING_CLIENT_SERVICE" - ErrSyncingClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" + ErrDeleteClientService druidv1alpha1.ErrorCode = "ERR_DELETE_CLIENT_SERVICE" + ErrSyncClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" ) type _resource struct { @@ -59,9 +61,9 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ctx.Logger.Info("synced", "resource", "client-service", "name", svc.Name, "result", result) } return druiderr.WrapError(err, - ErrSyncingClientService, + ErrSyncClientService, "Sync", - "Error during create or update of client service", + fmt.Sprintf("Error during create or update of client service for etcd: %v", etcd.GetNamespaceName()), ) } @@ -74,7 +76,7 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph } return druiderr.WrapError( err, - ErrDeletingClientService, + ErrDeleteClientService, "TriggerDelete", "Failed to delete client service", ) diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 484a9f02f..551ba5106 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -48,7 +48,7 @@ const ( func TestGetExistingResourceNames(t *testing.T) { internalErr := errors.New("test internal error") etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() - testcases := []struct { + testCases := []struct { name string svcExists bool getErr *apierrors.StatusError @@ -65,7 +65,7 @@ func TestGetExistingResourceNames(t *testing.T) { { "should return empty slice when service is not found", false, - apierrors.NewNotFound(corev1.Resource("services"), ""), + apierrors.NewNotFound(corev1.Resource("services"), etcd.GetClientServiceName()), nil, []string{}, }, @@ -81,7 +81,7 @@ func TestGetExistingResourceNames(t *testing.T) { g := NewWithT(t) t.Parallel() - for _, tc := range testcases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder() if tc.getErr != nil { @@ -134,7 +134,7 @@ func TestClientServiceSync(t *testing.T) { svcExists: false, setupFn: nil, expectError: &druiderr.DruidError{ - Code: ErrSyncingClientService, + Code: ErrSyncClientService, Cause: fmt.Errorf("fake get error"), Operation: "Sync", Message: "Error during create or update of client service", @@ -218,7 +218,7 @@ func TestClientServiceTriggerDelete(t *testing.T) { svcExists: true, setupFn: nil, expectError: &druiderr.DruidError{ - Code: ErrDeletingClientService, + Code: ErrDeleteClientService, Cause: errors.New("fake delete error"), Operation: "TriggerDelete", Message: "Failed to delete client service", diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 61101cf19..10df5a06d 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -1,7 +1,10 @@ package role import ( + "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" @@ -10,6 +13,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrSyncRole druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE" + ErrDeleteRole druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE" +) + type _resource struct { client client.Client } @@ -29,17 +37,33 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { role := emptyRole(etcd) - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { + result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { role.Labels = etcd.GetDefaultLabels() role.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} role.Rules = createPolicyRules() return nil }) - return err + if err == nil { + ctx.Logger.Info("synced", "resource", "role", "name", role.Name, "result", result) + } + return druiderr.WrapError(err, + ErrSyncRole, + "Sync", + fmt.Sprintf("Error during create or update of role %s for etcd: %v", etcd.GetRoleName(), etcd.GetNamespaceName()), + ) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return r.client.Delete(ctx, emptyRole(etcd)) + ctx.Logger.Info("Triggering delete of role") + err := r.client.Delete(ctx, emptyRole(etcd)) + if err == nil { + ctx.Logger.Info("deleted", "resource", "role", "name", etcd.GetRoleName()) + } + return druiderr.WrapError(err, + ErrDeleteRole, + "TriggerDelete", + fmt.Sprintf("Failed to delete role: %s for etcd: %v", etcd.GetRoleName(), etcd.GetNamespaceName()), + ) } func New(client client.Client) resource.Operator { diff --git a/test/sample/role.go b/test/sample/role.go new file mode 100644 index 000000000..46757c1b3 --- /dev/null +++ b/test/sample/role.go @@ -0,0 +1,39 @@ +package sample + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NewRole creates a new sample Role initializing it from the passed in etcd object. +func NewRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetRoleName(), + Namespace: etcd.Namespace, + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + Rules: createPolicyRules(), + } +} + +func createPolicyRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"statefulsets"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + } +} From 45c31e12974b007e9f8d6d11403958d14fcebc07 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 13:07:15 +0530 Subject: [PATCH 056/235] Removed duplicate import --- .../clientservice/clientservice_test.go | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 551ba5106..c0a105859 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -30,7 +30,7 @@ import ( testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" - "github.com/onsi/gomega" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -164,7 +164,7 @@ func TestClientServiceSync(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) if tc.expectError != nil { - g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err).To(HaveOccurred()) var druidErr *druiderr.DruidError g.Expect(errors.As(err, &druidErr)).To(BeTrue()) g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) @@ -173,7 +173,7 @@ func TestClientServiceSync(t *testing.T) { g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) } else { - g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred()) service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: etcd.GetClientServiceName(), @@ -181,7 +181,7 @@ func TestClientServiceSync(t *testing.T) { }, } err = cl.Get(opCtx, client.ObjectKeyFromObject(service), service) - g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred()) checkClientService(g, service, etcd) } }) @@ -247,7 +247,7 @@ func TestClientServiceTriggerDelete(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) if tc.expectError != nil { - g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err).To(HaveOccurred()) var druidErr *druiderr.DruidError g.Expect(errors.As(err, &druidErr)).To(BeTrue()) g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) @@ -256,18 +256,18 @@ func TestClientServiceTriggerDelete(t *testing.T) { g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) } else { - g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred()) serviceList := &corev1.List{} err = cl.List(opCtx, serviceList) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(serviceList.Items).To(gomega.BeEmpty()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(serviceList.Items).To(BeEmpty()) } }) } } -func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { +func checkClientService(g *WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) @@ -278,26 +278,26 @@ func checkClientService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) } - g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(svc.Annotations).To(gomega.Equal(expectedAnnotations)) - g.Expect(svc.Labels).To(gomega.Equal(expectedLabels)) - g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) - g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) - g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( - gomega.Equal(corev1.ServicePort{ + g.Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(svc.Annotations).To(Equal(expectedAnnotations)) + g.Expect(svc.Labels).To(Equal(expectedLabels)) + g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + g.Expect(svc.Spec.SessionAffinity).To(Equal(corev1.ServiceAffinityNone)) + g.Expect(svc.Spec.Selector).To(Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.Ports).To(ConsistOf( + Equal(corev1.ServicePort{ Name: "client", Protocol: corev1.ProtocolTCP, Port: clientPort, TargetPort: intstr.FromInt(int(clientPort)), }), - gomega.Equal(corev1.ServicePort{ + Equal(corev1.ServicePort{ Name: "server", Protocol: corev1.ProtocolTCP, Port: peerPort, TargetPort: intstr.FromInt(int(peerPort)), }), - gomega.Equal(corev1.ServicePort{ + Equal(corev1.ServicePort{ Name: "backuprestore", Protocol: corev1.ProtocolTCP, Port: backupPort, From 6d4a60df7ab119632a4d719f2692efee62fbf740 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 13:13:28 +0530 Subject: [PATCH 057/235] Enhance peer service operatior with error handling and logging, and update tests --- internal/operator/peerservice/peerservice.go | 30 +- .../operator/peerservice/peerservice_test.go | 396 ++++++++++-------- test/sample/service.go | 29 ++ 3 files changed, 275 insertions(+), 180 deletions(-) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 7a727cbb5..7fb2325a0 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -2,6 +2,7 @@ package peerservice import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -14,6 +15,11 @@ import ( const defaultServerPort = 2380 +const ( + ErrDeletingPeerService druidv1alpha1.ErrorCode = "ERR_DELETING_PEER_SERVICE" + ErrSyncingPeerService druidv1alpha1.ErrorCode = "ERR_SYNC_PEER_SERVICE" +) + type _resource struct { client client.Client } @@ -33,7 +39,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { svc := emptyPeerService(getObjectKey(etcd)) - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { + result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { svc.Labels = etcd.GetDefaultLabels() svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} svc.Spec.Type = corev1.ServiceTypeClusterIP @@ -44,11 +50,29 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) svc.Spec.Ports = getPorts(etcd) return nil }) - return err + if err == nil { + ctx.Logger.Info("synced", "resource", "peer-service", "name", svc.Name, "result", result) + } + return druiderr.WrapError(err, + ErrSyncingPeerService, + "Sync", + "Error during create or update of peer service", + ) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return client.IgnoreNotFound(r.client.Delete(ctx, emptyPeerService(getObjectKey(etcd)))) + objectKey := getObjectKey(etcd) + ctx.Logger.Info("Triggering delete of client service") + err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPeerService(objectKey))) + if err == nil { + ctx.Logger.Info("deleted", "resource", "peer-service", "name", objectKey.Name) + } + return druiderr.WrapError( + err, + ErrDeletingPeerService, + "TriggerDelete", + "Failed to delete peer service", + ) } func New(client client.Client) resource.Operator { diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 189a650ce..28c21e91c 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -16,165 +16,259 @@ package peerservice import ( "context" + "errors" "fmt" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + "github.com/gardener/etcd-druid/test/sample" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" - "github.com/golang/mock/gomock" - "github.com/onsi/gomega" + "github.com/google/uuid" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -// ------------------------ GetExistingResourceNames ------------------------ -func TestGetExistingResourceNames_WithExistingService(t *testing.T) { - g, ctx, etcd, _, op := setupWithFakeClient(t, true) - - err := op.Sync(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - names, err := op.GetExistingResourceNames(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(names).To(gomega.ContainElement(etcd.GetPeerServiceName())) - -} - -func TestGetExistingResourceNames_ServiceNotFound(t *testing.T) { - g, ctx, etcd, _, op := setupWithFakeClient(t, false) - - names, err := op.GetExistingResourceNames(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(names).To(gomega.BeEmpty()) -} - -func TestGetExistingResourceNames_WithError(t *testing.T) { - g, ctx, etcd, cl, op := setupWithMockClient(t) - cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() - - names, err := op.GetExistingResourceNames(ctx, etcd) - g.Expect(err).To(gomega.HaveOccurred()) - g.Expect(names).To(gomega.BeEmpty()) -} - -// ----------------------------------- Sync ----------------------------------- - -func TestSync_CreateNewService(t *testing.T) { - g, ctx, etcd, cl, op := setupWithFakeClient(t, false) - err := op.Sync(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetPeerServiceName(), - Namespace: etcd.Namespace, +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + internalErr := errors.New("test internal error") + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testcases := []struct { + name string + svcExists bool + getErr *apierrors.StatusError + expectedErr error + expectedServiceNames []string + }{ + { + "should return the existing service name", + true, + nil, + nil, + []string{etcd.GetPeerServiceName()}, + }, + { + "should return empty slice when service is not found", + false, + apierrors.NewNotFound(corev1.Resource("services"), ""), + nil, + []string{}, + }, + { + "should return error when get fails", + true, + apierrors.NewInternalError(internalErr), + apierrors.NewInternalError(internalErr), + nil, }, } - err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) - g.Expect(err).NotTo(gomega.HaveOccurred()) - checkPeerService(g, service, etcd) + g := NewWithT(t) + t.Parallel() + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.svcExists { + fakeClientBuilder.WithObjects(sample.NewPeerService(etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + } else { + g.Expect(err).To(BeNil()) + } + g.Expect(svcNames, tc.expectedServiceNames) + }) + } } -func TestSync_UpdateExistingService(t *testing.T) { - g, ctx, etcd, cl, op := setupWithFakeClient(t, true) - etcd.Spec.Etcd.ServerPort = nil - - err := op.Sync(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) +// ----------------------------------- Sync ----------------------------------- +func TestPeerServiceSync(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + svcExists bool + setupFn func(eb *testsample.EtcdBuilder) + expectError *druiderr.DruidError + getErr *apierrors.StatusError + }{ + { + name: "Create new service", + svcExists: false, + }, + { + name: "Update existing service", + svcExists: true, - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetPeerServiceName(), - Namespace: etcd.Namespace, + setupFn: func(eb *testsample.EtcdBuilder) { + eb.WithBackupPort(nil) + }, + }, + { + name: "With client error", + svcExists: false, + setupFn: nil, + expectError: &druiderr.DruidError{ + Code: ErrSyncingPeerService, + Cause: fmt.Errorf("fake get error"), + Operation: "Sync", + Message: "Error during create or update of peer service", + }, + getErr: apierrors.NewInternalError(errors.New("fake get error")), }, } - err = cl.Get(ctx, client.ObjectKeyFromObject(service), service) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(service).ToNot(gomega.BeNil()) - checkPeerService(g, service, etcd) -} -func TestSync_WithClientError(t *testing.T) { - g, ctx, etcd, cl, op := setupWithMockClient(t) - cl.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake get error")).AnyTimes() - cl.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake create error")).AnyTimes() - cl.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake update error")).AnyTimes() - - err := op.Sync(ctx, etcd) - g.Expect(err).To(gomega.HaveOccurred()) + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.setupFn != nil { + tc.setupFn(etcdBuilder) + } + etcd := etcdBuilder.Build() + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.svcExists { + fakeClientBuilder.WithObjects(sample.NewPeerService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, etcd) + if tc.expectError != nil { + g.Expect(err).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.getErr)).To(BeTrue()) + g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) + g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) + + } else { + g.Expect(err).NotTo(HaveOccurred()) + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetPeerServiceName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(opCtx, client.ObjectKeyFromObject(service), service) + g.Expect(err).NotTo(HaveOccurred()) + checkPeerService(g, service, etcd) + } + }) + } } // ----------------------------- TriggerDelete ------------------------------- - -func TestTriggerDelete_ExistingService(t *testing.T) { - g, ctx, etcd, cl, op := setupWithFakeClient(t, true) - err := op.TriggerDelete(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - serviceList := &corev1.List{} - err = cl.List(ctx, serviceList) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(serviceList.Items).To(gomega.BeEmpty()) -} - -func TestTriggerDelete_ServiceNotFound(t *testing.T) { - g, ctx, etcd, _, op := setupWithFakeClient(t, false) - - err := op.TriggerDelete(ctx, etcd) - g.Expect(err).NotTo(gomega.HaveOccurred()) -} - -func TestTriggerDelete_WithClientError(t *testing.T) { - g, ctx, etcd, cl, op := setupWithMockClient(t) - // Configure the mock client to return an error on delete operation - cl.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake delete error")).AnyTimes() - - err := op.TriggerDelete(ctx, etcd) - g.Expect(err).To(gomega.HaveOccurred()) -} - -// ---------------------------- Helper Functions ----------------------------- - -func sampleEtcd() *druidv1alpha1.Etcd { - return &druidv1alpha1.Etcd{ - - ObjectMeta: metav1.ObjectMeta{ - Name: "test-etcd", - Namespace: "test-namespace", - UID: "xxx-yyy-zzz", +func TestPeerServiceTriggerDelete(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + svcExists bool + setupFn func(eb *testsample.EtcdBuilder) + expectError *druiderr.DruidError + deleteErr *apierrors.StatusError + }{ + { + name: "Existing Service - Delete Operation", + svcExists: false, }, - Spec: druidv1alpha1.EtcdSpec{ - - Backup: druidv1alpha1.BackupSpec{ - Port: pointer.Int32(1111), + { + name: "Service Not Found - No Operation", + svcExists: true, + setupFn: func(eb *testsample.EtcdBuilder) { + eb.WithEtcdServerPort(nil) }, - Etcd: druidv1alpha1.EtcdConfig{ - ClientPort: pointer.Int32(2222), - ServerPort: pointer.Int32(3333), + }, + { + name: "Client Error on Delete - Returns Error", + svcExists: true, + setupFn: nil, + expectError: &druiderr.DruidError{ + Code: ErrDeletingPeerService, + Cause: errors.New("fake delete error"), + Operation: "TriggerDelete", + Message: "Failed to delete peer service", }, + deleteErr: apierrors.NewInternalError(errors.New("fake delete error")), }, } + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.setupFn != nil { + tc.setupFn(etcdBuilder) + } + etcd := etcdBuilder.Build() + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.deleteErr != nil { + fakeClientBuilder.WithDeleteError(tc.deleteErr) + } + if tc.svcExists { + fakeClientBuilder.WithObjects(sample.NewPeerService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.TriggerDelete(opCtx, etcd) + if tc.expectError != nil { + g.Expect(err).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.deleteErr)).To(BeTrue()) + g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) + g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) + + } else { + g.Expect(err).NotTo(HaveOccurred()) + serviceList := &corev1.List{} + err = cl.List(opCtx, serviceList) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(serviceList.Items).To(BeEmpty()) + + } + }) + } } -func checkPeerService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { +// ---------------------------- Helper Functions ----------------------------- +func checkPeerService(g *WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) - g.Expect(svc.OwnerReferences).To(gomega.Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(svc.Labels).To(gomega.Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.PublishNotReadyAddresses).To(gomega.BeTrue()) - g.Expect(svc.Spec.Type).To(gomega.Equal(corev1.ServiceTypeClusterIP)) - g.Expect(svc.Spec.ClusterIP).To(gomega.Equal(corev1.ClusterIPNone)) - g.Expect(svc.Spec.SessionAffinity).To(gomega.Equal(corev1.ServiceAffinityNone)) - g.Expect(svc.Spec.Selector).To(gomega.Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.Ports).To(gomega.ConsistOf( - gomega.Equal(corev1.ServicePort{ + g.Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(svc.Labels).To(Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.PublishNotReadyAddresses).To(BeTrue()) + g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + g.Expect(svc.Spec.ClusterIP).To(Equal(corev1.ClusterIPNone)) + g.Expect(svc.Spec.SessionAffinity).To(Equal(corev1.ServiceAffinityNone)) + g.Expect(svc.Spec.Selector).To(Equal(etcd.GetDefaultLabels())) + g.Expect(svc.Spec.Ports).To(ConsistOf( + Equal(corev1.ServicePort{ Name: "peer", Protocol: corev1.ProtocolTCP, Port: peerPort, @@ -182,55 +276,3 @@ func checkPeerService(g *gomega.WithT, svc *corev1.Service, etcd *druidv1alpha1. }), )) } - -func existingService(etcd *druidv1alpha1.Etcd) *corev1.Service { - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetPeerServiceName(), - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReferences: []metav1.OwnerReference{ - etcd.GetAsOwnerReference(), - }, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, - SessionAffinity: corev1.ServiceAffinityNone, - Selector: etcd.GetDefaultLabels(), - PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "peer", - Protocol: corev1.ProtocolTCP, - Port: *etcd.Spec.Etcd.ServerPort, - TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), - }, - }, - }, - } - return service -} - -func setupWithFakeClient(t *testing.T, withExistingService bool) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl client.Client, op resource.Operator) { - g = gomega.NewWithT(t) - ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") - etcd = sampleEtcd() - if withExistingService { - cl = fakeclient.NewClientBuilder().WithObjects(existingService(etcd)).Build() - } else { - - cl = fakeclient.NewClientBuilder().Build() - } - op = New(cl) - return -} - -func setupWithMockClient(t *testing.T) (g *gomega.WithT, ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, cl *mockclient.MockClient, op resource.Operator) { - g = gomega.NewWithT(t) - ctx = resource.NewOperatorContext(context.TODO(), logr.Logger{}, "xxx-yyyy-zzzz") - etcd = sampleEtcd() - cl = mockclient.NewMockClient(gomock.NewController(t)) - op = New(cl) - return -} diff --git a/test/sample/service.go b/test/sample/service.go index 3bdd4e96e..48bd02c14 100644 --- a/test/sample/service.go +++ b/test/sample/service.go @@ -31,6 +31,35 @@ func NewClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { } } +// NewPeerService creates a new sample peer service initializing it from the passed in etcd object. +func NewPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetPeerServiceName(), + Namespace: etcd.Namespace, + Labels: etcd.GetDefaultLabels(), + OwnerReferences: []metav1.OwnerReference{ + etcd.GetAsOwnerReference(), + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, + SessionAffinity: corev1.ServiceAffinityNone, + Selector: etcd.GetDefaultLabels(), + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "peer", + Protocol: corev1.ProtocolTCP, + Port: *etcd.Spec.Etcd.ServerPort, + TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), + }, + }, + }, + } +} + func getClientServicePorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) From 450a94691072c8cb0aff951d2acfe80606b3c93b Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 13:22:52 +0530 Subject: [PATCH 058/235] Standardize naming conventions for error codes in peer service. --- internal/operator/peerservice/peerservice.go | 8 ++++---- internal/operator/peerservice/peerservice_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 7fb2325a0..74f9a5c99 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -16,8 +16,8 @@ import ( const defaultServerPort = 2380 const ( - ErrDeletingPeerService druidv1alpha1.ErrorCode = "ERR_DELETING_PEER_SERVICE" - ErrSyncingPeerService druidv1alpha1.ErrorCode = "ERR_SYNC_PEER_SERVICE" + ErrDeletePeerService druidv1alpha1.ErrorCode = "ERR_DELETE_PEER_SERVICE" + ErrSyncPeerService druidv1alpha1.ErrorCode = "ERR_SYNC_PEER_SERVICE" ) type _resource struct { @@ -54,7 +54,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ctx.Logger.Info("synced", "resource", "peer-service", "name", svc.Name, "result", result) } return druiderr.WrapError(err, - ErrSyncingPeerService, + ErrSyncPeerService, "Sync", "Error during create or update of peer service", ) @@ -69,7 +69,7 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph } return druiderr.WrapError( err, - ErrDeletingPeerService, + ErrDeletePeerService, "TriggerDelete", "Failed to delete peer service", ) diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 28c21e91c..273922522 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -128,7 +128,7 @@ func TestPeerServiceSync(t *testing.T) { svcExists: false, setupFn: nil, expectError: &druiderr.DruidError{ - Code: ErrSyncingPeerService, + Code: ErrSyncPeerService, Cause: fmt.Errorf("fake get error"), Operation: "Sync", Message: "Error during create or update of peer service", @@ -208,7 +208,7 @@ func TestPeerServiceTriggerDelete(t *testing.T) { svcExists: true, setupFn: nil, expectError: &druiderr.DruidError{ - Code: ErrDeletingPeerService, + Code: ErrDeletePeerService, Cause: errors.New("fake delete error"), Operation: "TriggerDelete", Message: "Failed to delete peer service", From d431b5452ccd863b69c35e147f2b9a964e91b484 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Fri, 5 Jan 2024 17:08:49 +0530 Subject: [PATCH 059/235] Rebased with PR#737 --- internal/common/constants.go | 46 ++++-- .../etcdcopybackupstask/reconciler.go | 61 ++++---- .../etcdcopybackupstask/reconciler_test.go | 140 +++++++++--------- internal/operator/statefulset/builder.go | 104 ++----------- internal/utils/envvar.go | 112 ++++++++++++++ internal/utils/image_test.go | 12 +- internal/utils/statefulset_test.go | 3 +- test/sample/etcd.go | 6 +- test/sample/service.go | 8 +- test/utils/misc.go | 23 +++ 10 files changed, 293 insertions(+), 222 deletions(-) create mode 100644 internal/utils/envvar.go diff --git a/internal/common/constants.go b/internal/common/constants.go index 0882c630f..8a2571631 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -23,6 +23,8 @@ const ( EtcdWrapper = "etcd-wrapper" // BackupRestoreDistroless is the key for the etcd-backup-restore image in the image vector. BackupRestoreDistroless = "etcd-backup-restore-distroless" + // Alpine is the key for the alpine image in the image vector. + Alpine = "alpine" // ChartPath is the directory containing the default image vector file. ChartPath = "charts" // GardenerOwnedBy is a constant for an annotation on a resource that describes the owner resource. @@ -31,18 +33,34 @@ const ( GardenerOwnerType = "gardener.cloud/owner-type" // FinalizerName is the name of the etcd finalizer. FinalizerName = "druid.gardener.cloud/etcd-druid" - // STORAGE_CONTAINER is the environment variable key for the storage container. - STORAGE_CONTAINER = "STORAGE_CONTAINER" - // AWS_APPLICATION_CREDENTIALS is the environment variable key for AWS application credentials. - AWS_APPLICATION_CREDENTIALS = "AWS_APPLICATION_CREDENTIALS" - // AZURE_APPLICATION_CREDENTIALS is the environment variable key for Azure application credentials. - AZURE_APPLICATION_CREDENTIALS = "AZURE_APPLICATION_CREDENTIALS" - // GOOGLE_APPLICATION_CREDENTIALS is the environment variable key for Google application credentials. - GOOGLE_APPLICATION_CREDENTIALS = "GOOGLE_APPLICATION_CREDENTIALS" - // OPENSTACK_APPLICATION_CREDENTIALS is the environment variable key for OpenStack application credentials. - OPENSTACK_APPLICATION_CREDENTIALS = "OPENSTACK_APPLICATION_CREDENTIALS" - // OPENSHIFT_APPLICATION_CREDENTIALS is the environment variable key for OpenShift application credentials. - OPENSHIFT_APPLICATION_CREDENTIALS = "OPENSHIFT_APPLICATION_CREDENTIALS" - // ALICLOUD_APPLICATION_CREDENTIALS is the environment variable key for Alicloud application credentials. - ALICLOUD_APPLICATION_CREDENTIALS = "ALICLOUD_APPLICATION_CREDENTIALS" + + // EnvPodName is the environment variable key for the pod name. + EnvPodName = "POD_NAME" + // EnvPodNamespace is the environment variable key for the pod namespace. + EnvPodNamespace = "POD_NAMESPACE" + // EnvStorageContainer is the environment variable key for the storage container. + EnvStorageContainer = "STORAGE_CONTAINER" + // EnvSourceStorageContainer is the environment variable key for the source storage container. + EnvSourceStorageContainer = "SOURCE_STORAGE_CONTAINER" + + // EnvAWSApplicationCredentials is the environment variable key for AWS application credentials. + EnvAWSApplicationCredentials = "AWS_APPLICATION_CREDENTIALS" + // EnvAzureApplicationCredentials is the environment variable key for Azure application credentials. + EnvAzureApplicationCredentials = "AZURE_APPLICATION_CREDENTIALS" + // EnvGoogleApplicationCredentials is the environment variable key for Google application credentials. + EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS" + // EnvGoogleStorageAPIEndpoint is the environment variable key for Google storage API endpoint override. + EnvGoogleStorageAPIEndpoint = "GOOGLE_STORAGE_API_ENDPOINT" + // EnvOpenstackApplicationCredentials is the environment variable key for OpenStack application credentials. + EnvOpenstackApplicationCredentials = "OPENSTACK_APPLICATION_CREDENTIALS" + // EnvAlicloudApplicationCredentials is the environment variable key for Alicloud application credentials. + EnvAlicloudApplicationCredentials = "ALICLOUD_APPLICATION_CREDENTIALS" + // EnvOpenshiftApplicationCredentials is the environment variable key for OpenShift application credentials. + EnvOpenshiftApplicationCredentials = "OPENSHIFT_APPLICATION_CREDENTIALS" + // EnvECSEndpoint is the environment variable key for Dell ECS endpoint. + EnvECSEndpoint = "ECS_ENDPOINT" + // EnvECSAccessKeyID is the environment variable key for Dell ECS access key ID. + EnvECSAccessKeyID = "ECS_ACCESS_KEY_ID" + // EnvECSSecretAccessKey is the environment variable key for Dell ECS secret access key. + EnvECSSecretAccessKey = "ECS_SECRET_ACCESS_KEY" ) diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index e48612cdb..9c26ee86a 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -22,9 +22,9 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/controller/utils" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" - druidutils "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/etcd-druid/pkg/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -61,7 +61,7 @@ type Reconciler struct { // NewReconciler creates a new reconciler for EtcdCopyBackupsTask. func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := utils.CreateImageVector() + imageVector, err := ctrlutils.CreateImageVector() if err != nil { return nil, err } @@ -285,24 +285,24 @@ func getConditionType(jobConditionType batchv1.JobConditionType) druidv1alpha1.C } func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (*batchv1.Job, error) { - etcdBackupImage, err := druidutils.GetEtcdBackupRestoreImage(r.imageVector, r.Config.FeatureGates[features.UseEtcdWrapper]) + etcdBackupImage, err := utils.GetEtcdBackupRestoreImage(r.imageVector, r.Config.FeatureGates[features.UseEtcdWrapper]) if err != nil { return nil, err } - initContainerImage, err := druidutils.GetInitContainerImage(r.imageVector) + initContainerImage, err := utils.GetInitContainerImage(r.imageVector) if err != nil { return nil, err } targetStore := task.Spec.TargetStore - targetProvider, err := druidutils.StorageProviderFromInfraProvider(targetStore.Provider) + targetProvider, err := utils.StorageProviderFromInfraProvider(targetStore.Provider) if err != nil { return nil, err } sourceStore := task.Spec.SourceStore - sourceProvider, err := druidutils.StorageProviderFromInfraProvider(sourceStore.Provider) + sourceProvider, err := utils.StorageProviderFromInfraProvider(sourceStore.Provider) if err != nil { return nil, err } @@ -368,7 +368,7 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et } if r.Config.FeatureGates[features.UseEtcdWrapper] { - if targetProvider == druidutils.Local { + if targetProvider == utils.Local { // init container to change file permissions of the folders used as store to 65532 (nonroot) // used only with local provider job.Spec.Template.Spec.InitContainers = []corev1.Container{ @@ -440,9 +440,9 @@ func getVolumeNamePrefix(prefix string) string { // This function creates the necessary Volume configurations for various storage providers. func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1alpha1.StoreSpec, namespace, provider, prefix string) (volumes []corev1.Volume, err error) { switch provider { - case druidutils.Local: + case utils.Local: hostPathDirectory := corev1.HostPathDirectory - hostPathPrefix, err := druidutils.GetHostMountPathFromSecretRef(ctx, r.Client, r.logger, store, namespace) + hostPathPrefix, err := utils.GetHostMountPathFromSecretRef(ctx, r.Client, r.logger, store, namespace) if err != nil { return nil, err } @@ -455,7 +455,7 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a }, }, }) - case druidutils.GCS, druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + case utils.GCS, utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: if store.SecretRef == nil { err = fmt.Errorf("no secretRef is configured for backup %sstore", prefix) return @@ -478,7 +478,7 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a // This function creates the necessary Volume configurations for various storage providers and returns any errors encountered. func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volumeMountPrefix string, useEtcdWrapper bool) (volumeMounts []corev1.VolumeMount) { switch provider { - case druidutils.Local: + case utils.Local: if useEtcdWrapper { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: volumeMountPrefix + "host-storage", @@ -490,12 +490,12 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum MountPath: *store.Container, }) } - case druidutils.GCS: + case utils.GCS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", MountPath: "/var/." + getVolumeNamePrefix(volumeMountPrefix) + "gcp/", }) - case druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", MountPath: "/var/" + getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup/", @@ -504,32 +504,25 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum return } -func mapToEnvVar(name, value string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - Value: value, - } -} - // createEnvVarsFromStore generates a slice of environment variables for an EtcdCopyBackups job based on the given StoreSpec, // storeProvider, prefix, and volumePrefix. The prefix is used to differentiate between source and target environment variables. // This function creates the necessary environment variables for various storage providers and configurations. The generated // environment variables include storage container information and provider-specific credentials. func createEnvVarsFromStore(store *druidv1alpha1.StoreSpec, storeProvider, envKeyPrefix, volumePrefix string) (envVars []corev1.EnvVar) { - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.STORAGE_CONTAINER, *store.Container)) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvStorageContainer, *store.Container)) switch storeProvider { - case druidutils.S3: - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.AWS_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) - case druidutils.ABS: - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.AZURE_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) - case druidutils.GCS: - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.GOOGLE_APPLICATION_CREDENTIALS, "/var/."+volumePrefix+"gcp/serviceaccount.json")) - case druidutils.Swift: - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.OPENSTACK_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) - case druidutils.OCS: - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.OPENSHIFT_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) - case druidutils.OSS: - envVars = append(envVars, mapToEnvVar(envKeyPrefix+common.ALICLOUD_APPLICATION_CREDENTIALS, "/var/"+volumePrefix+"etcd-backup")) + case utils.S3: + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + case utils.ABS: + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + case utils.GCS: + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, "/var/."+volumePrefix+"gcp/serviceaccount.json")) + case utils.Swift: + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + case utils.OCS: + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + case utils.OSS: + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) } return envVars } diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 992fc11b6..aa500b798 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -19,6 +19,16 @@ import ( "fmt" "time" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/pkg/utils" + testutils "github.com/gardener/etcd-druid/test/utils" + + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/utils/imagevector" + . "github.com/gardener/gardener/pkg/utils/test/matchers" + "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -32,16 +42,6 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" - druidutils "github.com/gardener/etcd-druid/pkg/utils" - testutils "github.com/gardener/etcd-druid/test/utils" - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/imagevector" - . "github.com/gardener/gardener/pkg/utils/test/matchers" ) var _ = Describe("EtcdCopyBackupsTaskController", func() { @@ -307,8 +307,8 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Describe("#createJobArguments", func() { var ( - providerLocal = druidv1alpha1.StorageProvider(druidutils.Local) - providerS3 = druidv1alpha1.StorageProvider(druidutils.S3) + providerLocal = druidv1alpha1.StorageProvider(utils.Local) + providerS3 = druidv1alpha1.StorageProvider(utils.S3) task *druidv1alpha1.EtcdCopyBackupsTask expected = []string{ "copy", @@ -343,19 +343,19 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { }) It("should create the correct arguments", func() { - arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + arguments := createJobArgs(task, utils.Local, utils.S3) Expect(arguments).To(Equal(expected)) }) It("should include the max backup age in the arguments", func() { task.Spec.MaxBackupAge = pointer.Uint32(10) - arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + arguments := createJobArgs(task, utils.Local, utils.S3) Expect(arguments).To(Equal(append(expected, "--max-backup-age=10"))) }) It("should include the max number of backups in the arguments", func() { task.Spec.MaxBackups = pointer.Uint32(5) - arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + arguments := createJobArgs(task, utils.Local, utils.S3) Expect(arguments).To(Equal(append(expected, "--max-backups-to-copy=5"))) }) @@ -363,7 +363,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { task.Spec.WaitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ Enabled: true, } - arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + arguments := createJobArgs(task, utils.Local, utils.S3) Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true"))) }) @@ -372,7 +372,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Enabled: true, Timeout: &metav1.Duration{Duration: time.Minute}, } - arguments := createJobArgs(task, druidutils.Local, druidutils.S3) + arguments := createJobArgs(task, utils.Local, utils.S3) Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true", "--wait-for-final-snapshot-timeout=1m0s"))) }) }) @@ -386,12 +386,12 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { ) // Loop through different storage providers to test with for _, p := range []string{ - druidutils.ABS, - druidutils.GCS, - druidutils.S3, - druidutils.Swift, - druidutils.OSS, - druidutils.OCS, + utils.ABS, + utils.GCS, + utils.S3, + utils.Swift, + utils.OSS, + utils.OCS, } { Context(fmt.Sprintf("with provider #%s", p), func() { provider := p @@ -412,7 +412,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { } Context("with provider #Local", func() { BeforeEach(func() { - storageProvider := druidv1alpha1.StorageProvider(druidutils.Local) + storageProvider := druidv1alpha1.StorageProvider(utils.Local) storeSpec = &druidv1alpha1.StoreSpec{ Container: &container, Provider: &storageProvider, @@ -420,8 +420,8 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { }) It("should create the correct env vars", func() { - envVars := createEnvVarsFromStore(storeSpec, druidutils.Local, envKeyPrefix, volumePrefix) - checkEnvVars(envVars, druidutils.Local, container, envKeyPrefix, volumePrefix) + envVars := createEnvVarsFromStore(storeSpec, utils.Local, envKeyPrefix, volumePrefix) + checkEnvVars(envVars, utils.Local, container, envKeyPrefix, volumePrefix) }) }) @@ -434,13 +434,13 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { ) // Loop through different storage providers to test with for _, p := range []string{ - druidutils.Local, - druidutils.ABS, - druidutils.GCS, - druidutils.S3, - druidutils.Swift, - druidutils.OSS, - druidutils.OCS, + utils.Local, + utils.ABS, + utils.GCS, + utils.S3, + utils.Swift, + utils.OSS, + utils.OCS, } { Context(fmt.Sprintf("with provider #%s", p), func() { provider := p @@ -460,13 +460,13 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { expectedMountName := "" switch provider { - case druidutils.Local: + case utils.Local: expectedMountName = volumeMountPrefix + "host-storage" expectedMountPath = *storeSpec.Container - case druidutils.GCS: + case utils.GCS: expectedMountName = volumeMountPrefix + "etcd-backup" expectedMountPath = "/var/." + volumeMountPrefix + "gcp/" - case druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: expectedMountName = volumeMountPrefix + "etcd-backup" expectedMountPath = "/var/" + volumeMountPrefix + "etcd-backup/" default: @@ -515,7 +515,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { It("should create the correct volumes when secret data hostPath is set", func() { secret.Data = map[string][]byte{ - druidutils.EtcdBackupSecretHostPath: []byte("/test/hostPath"), + utils.EtcdBackupSecretHostPath: []byte("/test/hostPath"), } Expect(fakeClient.Create(ctx, secret)).To(Succeed()) @@ -542,7 +542,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { hostPathVolumeSource := volumes[0].VolumeSource.HostPath Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal(druidutils.LocalProviderDefaultMountPath + "/" + *store.Container)) + Expect(hostPathVolumeSource.Path).To(Equal(utils.LocalProviderDefaultMountPath + "/" + *store.Container)) Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) }) @@ -557,7 +557,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { hostPathVolumeSource := volumes[0].VolumeSource.HostPath Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal(druidutils.LocalProviderDefaultMountPath + "/" + *store.Container)) + Expect(hostPathVolumeSource.Path).To(Equal(utils.LocalProviderDefaultMountPath + "/" + *store.Container)) Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) }) }) @@ -578,12 +578,12 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { // Loop through different storage providers to test with for _, p := range []string{ - druidutils.ABS, - druidutils.GCS, - druidutils.S3, - druidutils.Swift, - druidutils.OSS, - druidutils.OCS, + utils.ABS, + utils.GCS, + utils.S3, + utils.Swift, + utils.OSS, + utils.OCS, } { Context(fmt.Sprintf("#%s", p), func() { BeforeEach(func() { @@ -684,24 +684,24 @@ func addDeletionTimestampToTask(ctx context.Context, task *druidv1alpha1.EtcdCop func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefix, volumePrefix string) { expected := []corev1.EnvVar{ { - Name: envKeyPrefix + "STORAGE_CONTAINER", + Name: envKeyPrefix + common.EnvStorageContainer, Value: container, }} mapToEnvVarKey := map[string]string{ - druidutils.S3: envKeyPrefix + common.AWS_APPLICATION_CREDENTIALS, - druidutils.ABS: envKeyPrefix + common.AZURE_APPLICATION_CREDENTIALS, - druidutils.GCS: envKeyPrefix + common.GOOGLE_APPLICATION_CREDENTIALS, - druidutils.Swift: envKeyPrefix + common.OPENSTACK_APPLICATION_CREDENTIALS, - druidutils.OCS: envKeyPrefix + common.OPENSHIFT_APPLICATION_CREDENTIALS, - druidutils.OSS: envKeyPrefix + common.ALICLOUD_APPLICATION_CREDENTIALS, + utils.S3: envKeyPrefix + common.EnvAWSApplicationCredentials, + utils.ABS: envKeyPrefix + common.EnvAzureApplicationCredentials, + utils.GCS: envKeyPrefix + common.EnvGoogleApplicationCredentials, + utils.Swift: envKeyPrefix + common.EnvOpenstackApplicationCredentials, + utils.OCS: envKeyPrefix + common.EnvOpenshiftApplicationCredentials, + utils.OSS: envKeyPrefix + common.EnvAlicloudApplicationCredentials, } switch storeProvider { - case druidutils.S3, druidutils.ABS, druidutils.Swift, druidutils.OCS, druidutils.OSS: + case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], Value: "/var/" + volumePrefix + "etcd-backup", }) - case druidutils.GCS: + case utils.GCS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], Value: "/var/." + volumePrefix + "gcp/serviceaccount.json", @@ -807,14 +807,14 @@ func getArgElements(task *druidv1alpha1.EtcdCopyBackupsTask, sourceProvider, tar func getEnvElements(task *druidv1alpha1.EtcdCopyBackupsTask) Elements { elements := Elements{} if task.Spec.TargetStore.Container != nil && *task.Spec.TargetStore.Container != "" { - elements["STORAGE_CONTAINER"] = MatchFields(IgnoreExtras, Fields{ - "Name": Equal("STORAGE_CONTAINER"), + elements[common.EnvStorageContainer] = MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.EnvStorageContainer), "Value": Equal(*task.Spec.TargetStore.Container), }) } if task.Spec.SourceStore.Container != nil && *task.Spec.SourceStore.Container != "" { - elements["SOURCE_STORAGE_CONTAINER"] = MatchFields(IgnoreExtras, Fields{ - "Name": Equal("SOURCE_STORAGE_CONTAINER"), + elements[common.EnvSourceStorageContainer] = MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.EnvSourceStorageContainer), "Value": Equal(*task.Spec.SourceStore.Container), }) } @@ -868,43 +868,43 @@ func getProviderEnvElements(storeProvider, prefix, volumePrefix string) Elements switch storeProvider { case "S3": return Elements{ - prefix + "AWS_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + "AWS_APPLICATION_CREDENTIALS"), + prefix + common.EnvAWSApplicationCredentials: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + common.EnvAWSApplicationCredentials), "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } case "ABS": return Elements{ - prefix + "AZURE_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + "AZURE_APPLICATION_CREDENTIALS"), + prefix + common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + common.EnvAzureApplicationCredentials), "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } case "GCS": return Elements{ - prefix + "GOOGLE_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + "GOOGLE_APPLICATION_CREDENTIALS"), + prefix + common.EnvGoogleApplicationCredentials: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + common.EnvGoogleApplicationCredentials), "Value": Equal(fmt.Sprintf("/var/.%sgcp/serviceaccount.json", volumePrefix)), }), } case "Swift": return Elements{ - prefix + "OPENSTACK_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + "OPENSTACK_APPLICATION_CREDENTIALS"), + prefix + common.EnvOpenstackApplicationCredentials: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + common.EnvOpenstackApplicationCredentials), "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } case "OSS": return Elements{ - prefix + "ALICLOUD_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + "ALICLOUD_APPLICATION_CREDENTIALS"), + prefix + common.EnvAlicloudApplicationCredentials: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + common.EnvAlicloudApplicationCredentials), "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } case "OCS": return Elements{ - prefix + "OPENSHIFT_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + "OPENSHIFT_APPLICATION_CREDENTIALS"), + prefix + common.EnvOpenshiftApplicationCredentials: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(prefix + common.EnvOpenshiftApplicationCredentials), "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 6b480304d..86ea470d5 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -121,6 +121,12 @@ func (b *stsBuilder) createStatefulSetSpec(ctx resource.OperatorContext) error { if err != nil { return err } + + backupRestoreContainer, err := b.getBackupRestoreContainer() + if err != nil { + return err + } + b.sts.Spec = appsv1.StatefulSetSpec{ Replicas: pointer.Int32(b.replicas), Selector: &metav1.LabelSelector{ @@ -142,7 +148,7 @@ func (b *stsBuilder) createStatefulSetSpec(ctx resource.OperatorContext) error { InitContainers: b.getPodInitContainers(), Containers: []corev1.Container{ b.getEtcdContainer(), - b.getBackupRestoreContainer(), + backupRestoreContainer, }, SecurityContext: b.getPodSecurityContext(), Affinity: b.etcd.Spec.SchedulingConstraints.Affinity, @@ -324,7 +330,11 @@ func (b *stsBuilder) getEtcdContainer() corev1.Container { } } -func (b *stsBuilder) getBackupRestoreContainer() corev1.Container { +func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { + env, err := utils.GetBackupRestoreContainerEnvVars(b.etcd.Spec.Backup.Store) + if err != nil { + return corev1.Container{}, err + } return corev1.Container{ Name: "backup-restore", Image: b.etcdBackupRestoreImage, @@ -337,7 +347,7 @@ func (b *stsBuilder) getBackupRestoreContainer() corev1.Container { ContainerPort: b.backupPort, }, }, - Env: b.getBackupRestorContainerEnvVars(), + Env: env, Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Backup.Resources, defaultResourceRequirements), VolumeMounts: b.getBackupRestoreContainerVolumeMounts(), SecurityContext: &corev1.SecurityContext{ @@ -347,8 +357,7 @@ func (b *stsBuilder) getBackupRestoreContainer() corev1.Container { }, }, }, - } - + }, nil } func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { @@ -593,59 +602,6 @@ func (b *stsBuilder) getEtcdContainerEnvVars() []corev1.EnvVar { } } -func (b *stsBuilder) getBackupRestorContainerEnvVars() []corev1.EnvVar { - var ( - env []corev1.EnvVar - storageContainer string - storeValues = b.etcd.Spec.Backup.Store - ) - - if b.etcd.Spec.Backup.Store != nil { - storageContainer = pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, "") - } - - env = append(env, getEnvVarFromField("POD_NAME", "metadata.name")) - env = append(env, getEnvVarFromField("POD_NAMESPACE", "metadata.namespace")) - - if storeValues == nil { - return env - } - - provider, err := utils.StorageProviderFromInfraProvider(b.etcd.Spec.Backup.Store.Provider) - if err != nil { - return env - } - env = append(env, getEnvVarFromValue("STORAGE_CONTAINER", storageContainer)) - - const credentialsMountPath = "/var/etcd-backup" - switch provider { - case utils.S3: - env = append(env, getEnvVarFromValue("AWS_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.ABS: - env = append(env, getEnvVarFromValue("AZURE_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.GCS: - env = append(env, getEnvVarFromValue("GOOGLE_APPLICATION_CREDENTIALS", "/var/.gcp/serviceaccount.json")) - - case utils.Swift: - env = append(env, getEnvVarFromValue("OPENSTACK_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.OSS: - env = append(env, getEnvVarFromValue("ALICLOUD_APPLICATION_CREDENTIALS", credentialsMountPath)) - - case utils.ECS: - env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) - env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) - env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) - - case utils.OCS: - env = append(env, getEnvVarFromValue("OPENSHIFT_APPLICATION_CREDENTIALS", credentialsMountPath)) - } - - return env -} - func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { if !b.useEtcdWrapper { return nil @@ -824,35 +780,3 @@ func getBackupStoreProvider(etcd *druidv1alpha1.Etcd) (*string, error) { } return &provider, nil } - -func getEnvVarFromValue(name, value string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - Value: value, - } -} - -func getEnvVarFromField(name, fieldPath string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: fieldPath, - }, - }, - } -} - -func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, - }, - Key: secretKey, - }, - }, - } -} diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go new file mode 100644 index 000000000..a3d997635 --- /dev/null +++ b/internal/utils/envvar.go @@ -0,0 +1,112 @@ +package utils + +import ( + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/common" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +// GetEnvVarFromValue returns environment variable object with the provided name and value +func GetEnvVarFromValue(name, value string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + Value: value, + } +} + +// GetEnvVarFromFieldPath returns environment variable object with provided name and value from field path +func GetEnvVarFromFieldPath(name, fieldPath string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: fieldPath, + }, + }, + } +} + +// GetEnvVarFromSecret returns environment variable object with provided name and optional value from secret +func GetEnvVarFromSecret(name, secretName, secretKey string, optional bool) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + Key: secretKey, + Optional: pointer.Bool(optional), + }, + }, + } +} + +// GetProviderEnvVars returns provider-specific environment variables for the given store +func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { + var envVars []corev1.EnvVar + + provider, err := StorageProviderFromInfraProvider(store.Provider) + if err != nil { + return nil, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") + } + + const credentialsMountPath = "/var/etcd-backup" + switch provider { + case S3: + envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, credentialsMountPath)) + + case ABS: + envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, credentialsMountPath)) + + case GCS: + envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, "/var/.gcp/serviceaccount.json")) + envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) + + case Swift: + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, credentialsMountPath)) + + case OSS: + envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, credentialsMountPath)) + + case ECS: + if store.SecretRef == nil { + return nil, fmt.Errorf("no secretRef could be configured for backup store of ECS") + } + envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSEndpoint, store.SecretRef.Name, "endpoint", false)) + envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSAccessKeyID, store.SecretRef.Name, "accessKeyID", false)) + envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) + + case OCS: + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, credentialsMountPath)) + } + + return envVars, nil +} + +// GetBackupRestoreContainerEnvVars returns backup-restore container environment variables for the given store +func GetBackupRestoreContainerEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { + var envVars []corev1.EnvVar + + envVars = append(envVars, GetEnvVarFromFieldPath(common.EnvPodName, "metadata.name")) + envVars = append(envVars, GetEnvVarFromFieldPath(common.EnvPodNamespace, "metadata.namespace")) + + if store == nil { + return envVars, nil + } + + storageContainer := pointer.StringDeref(store.Container, "") + envVars = append(envVars, GetEnvVarFromValue(common.EnvStorageContainer, storageContainer)) + + providerEnvVars, err := GetProviderEnvVars(store) + if err != nil { + return nil, err + } + envVars = append(envVars, providerEnvVars...) + + return envVars, nil +} diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index 43aae31bc..1ce4923c1 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -17,7 +17,7 @@ package utils import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" - testutils "github.com/gardener/etcd-druid/test/utils" + testsample "github.com/gardener/etcd-druid/test/sample" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/ginkgo/v2" @@ -38,7 +38,7 @@ var _ = Describe("Image retrieval tests", func() { ) It("etcd spec defines etcd and backup-restore images", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() imageVector = createImageVector(true, true, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) Expect(err).To(BeNil()) @@ -52,7 +52,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec has no image defined and image vector has both images set", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil @@ -73,7 +73,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil imageVector = createImageVector(true, false, false, false) @@ -91,7 +91,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("both spec and image vector do not have backup-restore image", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Backup.Image = nil imageVector = createImageVector(true, false, false, false) @@ -103,7 +103,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec has no images defined, image vector has all images, and useEtcdWrapper feature gate is turned on", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index dd5d2a70b..8049aab43 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -19,6 +19,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/client/kubernetes" + testsample "github.com/gardener/etcd-druid/test/sample" "github.com/gardener/etcd-druid/test/utils" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -84,7 +85,7 @@ var _ = Describe("tests for statefulset utility functions", func() { BeforeEach(func() { ctx = context.TODO() stsListToCleanup = &appsv1.StatefulSetList{} - etcd = utils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() + etcd = testsample.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() }) AfterEach(func() { diff --git a/test/sample/etcd.go b/test/sample/etcd.go index 4add6d27e..aaf3fb7c8 100644 --- a/test/sample/etcd.go +++ b/test/sample/etcd.go @@ -18,7 +18,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" + testutils "github.com/gardener/etcd-druid/test/utils" corev1 "k8s.io/api/core/v1" @@ -253,7 +253,7 @@ func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *Et eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} } - eb.etcd.Spec.Etcd.ClientService.Labels = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) + eb.etcd.Spec.Etcd.ClientService.Labels = testutils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) return eb } @@ -266,7 +266,7 @@ func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]s eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} } - eb.etcd.Spec.Etcd.ClientService.Annotations = utils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) + eb.etcd.Spec.Etcd.ClientService.Annotations = testutils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) return eb } diff --git a/test/sample/service.go b/test/sample/service.go index 48bd02c14..b6972e53d 100644 --- a/test/sample/service.go +++ b/test/sample/service.go @@ -2,7 +2,7 @@ package sample import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" + testutils "github.com/gardener/etcd-druid/test/utils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -61,9 +61,9 @@ func NewPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { } func getClientServicePorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + backupPort := testutils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) + clientPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) + peerPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) return []corev1.ServicePort{ { Name: "client", diff --git a/test/utils/misc.go b/test/utils/misc.go index f1ff0ffb8..7ec636264 100644 --- a/test/utils/misc.go +++ b/test/utils/misc.go @@ -5,6 +5,7 @@ package utils import ( + "maps" "os" . "github.com/onsi/gomega" @@ -49,3 +50,25 @@ func SwitchDirectory(path string) func() { } } } + +// MergeMaps merges the contents of maps. All maps will be processed in the order +// in which they are sent. For overlapping keys across source maps, value in the merged map +// for this key will be from the last occurrence of the key-value. +func MergeMaps[K comparable, V any](sourceMaps ...map[K]V) map[K]V { + if sourceMaps == nil { + return nil + } + merged := make(map[K]V) + for _, m := range sourceMaps { + maps.Copy(merged, m) + } + return merged +} + +// TypeDeref dereferences a pointer to a type if it is not nil, else it returns the default value. +func TypeDeref[T any](val *T, defaultVal T) T { + if val != nil { + return *val + } + return defaultVal +} From f87b9bd1c3c6f030e856a474ba33267a31ee89be Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 5 Jan 2024 17:16:28 +0530 Subject: [PATCH 060/235] added unit test for role operator and added functions to fake client --- internal/operator/role/role.go | 2 +- internal/operator/role/role_test.go | 249 ++++++++++++++++++++++++++++ test/sample/etcd.go | 7 + test/utils/fakeclient.go | 47 ++++-- 4 files changed, 294 insertions(+), 11 deletions(-) diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 10df5a06d..1355fbd7f 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -55,7 +55,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of role") - err := r.client.Delete(ctx, emptyRole(etcd)) + err := client.IgnoreNotFound(r.client.Delete(ctx, emptyRole(etcd))) if err == nil { ctx.Logger.Info("deleted", "resource", "role", "name", etcd.GetRoleName()) } diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index e82991293..32a74d38d 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -1 +1,250 @@ package role + +import ( + "context" + "errors" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) + +var ( + internalErr = errors.New("test internal error") +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testCases := []struct { + name string + roleExists bool + getErr *apierrors.StatusError + expectedErr *apierrors.StatusError + expectedRoleNames []string + }{ + { + "should return the existing role name", + true, + nil, + nil, + []string{etcd.GetRoleName()}, + }, + { + "should return empty slice when role is not found", + false, + apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleName()), + nil, + []string{}, + }, + { + "should return error when get fails", + true, + apierrors.NewInternalError(internalErr), + apierrors.NewInternalError(internalErr), + nil, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.roleExists { + fakeClientBuilder.WithObjects(testsample.NewRole(etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + roleNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + } else { + g.Expect(err).To(BeNil()) + } + g.Expect(roleNames, tc.expectedRoleNames) + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSync(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + internalStatusErr := apierrors.NewInternalError(internalErr) + testCases := []struct { + name string + roleExists bool + getErr *apierrors.StatusError + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "create role when none exists", + roleExists: false, + }, + { + name: "create role fails when client create fails", + roleExists: false, + createErr: internalStatusErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncRole, + Cause: internalStatusErr, + Operation: "Sync", + }, + }, + { + name: "create role fails when get errors out", + roleExists: false, + getErr: internalStatusErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncRole, + Cause: internalStatusErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder(). + WithGetError(tc.getErr). + WithCreateError(tc.createErr) + if tc.roleExists { + fakeClientBuilder.WithObjects(testsample.NewRole(etcd)) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(err).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) + g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + existingRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetRoleName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(context.Background(), client.ObjectKeyFromObject(existingRole), existingRole) + g.Expect(err).ToNot(HaveOccurred()) + checkRole(g, existingRole, etcd) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + internalStatusErr := apierrors.NewInternalError(internalErr) + testCases := []struct { + name string + roleExists bool + deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "successfully delete existing role", + roleExists: true, + }, + { + name: "delete fails due to failing client delete", + roleExists: true, + deleteErr: internalStatusErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeleteRole, + Cause: internalStatusErr, + Operation: "TriggerDelete", + }, + }, + { + name: "delete is a no-op if role does not exist", + roleExists: false, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.deleteErr != nil { + fakeClientBuilder.WithDeleteError(tc.deleteErr) + } + if tc.roleExists { + fakeClientBuilder.WithObjects(testsample.NewRole(etcd)) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.TriggerDelete(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(err).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) + g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + existingRole := rbacv1.Role{} + err = cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace}, &existingRole) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- +func checkRole(g *WithT, role *rbacv1.Role, etcd *druidv1alpha1.Etcd) { + g.Expect(role.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(role.Labels).To(Equal(etcd.GetDefaultLabels())) + g.Expect(role.Rules).To(ConsistOf( + rbacv1.PolicyRule{ + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"apps"}, + Resources: []string{"statefulsets"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + )) +} diff --git a/test/sample/etcd.go b/test/sample/etcd.go index aaf3fb7c8..c34616fd3 100644 --- a/test/sample/etcd.go +++ b/test/sample/etcd.go @@ -286,6 +286,13 @@ func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { return eb } +// WithLabels merges the labels that already exists with the ones that are passed to this method. If an entry is +// already present in the existing labels then it gets overwritten with the new value present in the passed in labels. +func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { + utils.MergeMaps[string, string](eb.etcd.Labels, labels) + return eb +} + func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { return eb.etcd } diff --git a/test/utils/fakeclient.go b/test/utils/fakeclient.go index 8cb2acf28..f7433d551 100644 --- a/test/utils/fakeclient.go +++ b/test/utils/fakeclient.go @@ -3,6 +3,7 @@ package utils import ( "context" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -11,10 +12,10 @@ import ( // provides an ability to set custom errors for operations supported on the client. type FakeClientBuilder struct { clientBuilder *fakeclient.ClientBuilder - getErr error - createErr error - patchErr error - deleteErr error + getErr *apierrors.StatusError + createErr *apierrors.StatusError + patchErr *apierrors.StatusError + deleteErr *apierrors.StatusError } // NewFakeClientBuilder creates a new FakeClientBuilder. @@ -31,13 +32,25 @@ func (b *FakeClientBuilder) WithObjects(objs ...client.Object) *FakeClientBuilde } // WithGetError sets the error that should be returned when a Get request is made on the fake client. -func (b *FakeClientBuilder) WithGetError(err error) *FakeClientBuilder { +func (b *FakeClientBuilder) WithGetError(err *apierrors.StatusError) *FakeClientBuilder { b.getErr = err return b } +// WithCreateError sets the error that should be returned when a Create request is made on the fake client. +func (b *FakeClientBuilder) WithCreateError(err *apierrors.StatusError) *FakeClientBuilder { + b.createErr = err + return b +} + +// WithPatchError sets the error that should be returned when a Patch request is made on the fake client. +func (b *FakeClientBuilder) WithPatchError(err *apierrors.StatusError) *FakeClientBuilder { + b.patchErr = err + return b +} + // WithDeleteError sets the error that should be returned when a Delete request is made on the fake client. -func (b *FakeClientBuilder) WithDeleteError(err error) *FakeClientBuilder { +func (b *FakeClientBuilder) WithDeleteError(err *apierrors.StatusError) *FakeClientBuilder { b.deleteErr = err return b } @@ -55,10 +68,10 @@ func (b *FakeClientBuilder) Build() client.WithWatch { type fakeClient struct { client.WithWatch - getErr error - createErr error - patchErr error - deleteErr error + getErr *apierrors.StatusError + createErr *apierrors.StatusError + patchErr *apierrors.StatusError + deleteErr *apierrors.StatusError } // Get overwrites the fake client Get implementation with a capability to return any configured error. @@ -76,3 +89,17 @@ func (f *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie } return f.WithWatch.Delete(ctx, obj, opts...) } + +func (f *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if f.patchErr != nil { + return f.patchErr + } + return f.WithWatch.Patch(ctx, obj, patch, opts...) +} + +func (f *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if f.createErr != nil { + return f.createErr + } + return f.WithWatch.Create(ctx, obj, opts...) +} From 08b02d170b6a40ab08c405b751eb2137e5fdafdf Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sat, 6 Jan 2024 15:28:04 +0530 Subject: [PATCH 061/235] added rolebinding tests --- internal/operator/rolebinding/rolebinding.go | 31 ++- .../operator/rolebinding/rolebinding_test.go | 246 ++++++++++++++++++ test/sample/etcd.go | 2 +- test/sample/rolebinding.go | 29 +++ 4 files changed, 303 insertions(+), 5 deletions(-) create mode 100644 internal/operator/rolebinding/rolebinding_test.go create mode 100644 test/sample/rolebinding.go diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index f216ee5cd..22a7c2a2b 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -1,7 +1,10 @@ package rolebinding import ( + "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" @@ -10,6 +13,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrSyncRoleBinding druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE_BINDING" + ErrDeleteRoleBinding druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE_BINDING" +) + type _resource struct { client client.Client } @@ -40,7 +48,7 @@ func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { rb := emptyRoleBinding(etcd) - opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { + result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { rb.Labels = etcd.GetDefaultLabels() rb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} rb.RoleRef = rbacv1.RoleRef{ @@ -57,12 +65,27 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } return nil }) - ctx.Logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) - return err + if err == nil { + ctx.Logger.Info("synced", "resource", "role", "name", rb.Name, "result", result) + } + return druiderr.WrapError(err, + ErrSyncRoleBinding, + "Sync", + fmt.Sprintf("Error during create or update of role-binding %s for etcd: %v", etcd.GetRoleBindingName(), etcd.GetNamespaceName()), + ) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) + ctx.Logger.Info("Triggering delete of role") + err := client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) + if err == nil { + ctx.Logger.Info("deleted", "resource", "role-binding", "name", etcd.GetRoleBindingName()) + } + return druiderr.WrapError(err, + ErrDeleteRoleBinding, + "TriggerDelete", + fmt.Sprintf("Failed to delete role-binding: %s for etcd: %v", etcd.GetRoleBindingName(), etcd.GetNamespaceName()), + ) } func New(client client.Client) resource.Operator { diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go new file mode 100644 index 000000000..19757e28c --- /dev/null +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -0,0 +1,246 @@ +package rolebinding + +import ( + "context" + "errors" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/gomega" +) + +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) + +var ( + internalErr = errors.New("test internal error") +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testCases := []struct { + name string + roleBindingExists bool + getErr *apierrors.StatusError + expectedErr *apierrors.StatusError + expectedRoleBindingNames []string + }{ + { + "should return the existing role binding name", + true, + nil, + nil, + []string{etcd.GetRoleBindingName()}, + }, + { + "should return empty slice when role binding is not found", + false, + apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleBindingName()), + nil, + []string{}, + }, + { + "should return error when get fails", + true, + apierrors.NewInternalError(internalErr), + apierrors.NewInternalError(internalErr), + nil, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.roleBindingExists { + fakeClientBuilder.WithObjects(testsample.NewRoleBinding(etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + roleBindingNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + } else { + g.Expect(err).To(BeNil()) + } + g.Expect(roleBindingNames, tc.expectedRoleBindingNames) + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSync(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + internalStatusErr := apierrors.NewInternalError(internalErr) + testCases := []struct { + name string + roleBindingExists bool + getErr *apierrors.StatusError + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "create role when none exists", + roleBindingExists: false, + }, + { + name: "create role fails when client create fails", + roleBindingExists: false, + createErr: internalStatusErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncRoleBinding, + Cause: internalStatusErr, + Operation: "Sync", + }, + }, + { + name: "create role fails when get errors out", + roleBindingExists: false, + getErr: internalStatusErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncRoleBinding, + Cause: internalStatusErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder(). + WithGetError(tc.getErr). + WithCreateError(tc.createErr) + if tc.roleBindingExists { + fakeClientBuilder.WithObjects(testsample.NewRoleBinding(etcd)) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(err).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) + g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + existingRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetRoleBindingName(), + Namespace: etcd.Namespace, + }, + } + err = cl.Get(context.Background(), client.ObjectKeyFromObject(existingRoleBinding), existingRoleBinding) + g.Expect(err).ToNot(HaveOccurred()) + checkRoleBinding(g, existingRoleBinding, etcd) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + internalStatusErr := apierrors.NewInternalError(internalErr) + testCases := []struct { + name string + roleExists bool + deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "successfully delete existing role", + roleExists: true, + }, + { + name: "delete fails due to failing client delete", + roleExists: true, + deleteErr: internalStatusErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeleteRoleBinding, + Cause: internalStatusErr, + Operation: "TriggerDelete", + }, + }, + { + name: "delete is a no-op if role does not exist", + roleExists: false, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.deleteErr != nil { + fakeClientBuilder.WithDeleteError(tc.deleteErr) + } + if tc.roleExists { + fakeClientBuilder.WithObjects(testsample.NewRoleBinding(etcd)) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.TriggerDelete(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(err).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) + g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) + g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + existingRoleBinding := rbacv1.RoleBinding{} + err = cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace}, &existingRoleBinding) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- +func checkRoleBinding(g *WithT, roleBinding *rbacv1.RoleBinding, etcd *druidv1alpha1.Etcd) { + g.Expect(roleBinding.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(roleBinding.Labels).To(Equal(etcd.GetDefaultLabels())) + g.Expect(roleBinding.RoleRef).To(Equal(rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: etcd.GetRoleName(), + })) + g.Expect(roleBinding.Subjects).To(ConsistOf( + rbacv1.Subject{ + Kind: "ServiceAccount", + Name: etcd.GetServiceAccountName(), + Namespace: etcd.Namespace, + }, + )) +} diff --git a/test/sample/etcd.go b/test/sample/etcd.go index c34616fd3..4ec452c11 100644 --- a/test/sample/etcd.go +++ b/test/sample/etcd.go @@ -289,7 +289,7 @@ func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { // WithLabels merges the labels that already exists with the ones that are passed to this method. If an entry is // already present in the existing labels then it gets overwritten with the new value present in the passed in labels. func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { - utils.MergeMaps[string, string](eb.etcd.Labels, labels) + testutils.MergeMaps[string, string](eb.etcd.Labels, labels) return eb } diff --git a/test/sample/rolebinding.go b/test/sample/rolebinding.go new file mode 100644 index 000000000..bafa3c3b9 --- /dev/null +++ b/test/sample/rolebinding.go @@ -0,0 +1,29 @@ +package sample + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func NewRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetRoleBindingName(), + Namespace: etcd.Namespace, + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: etcd.GetServiceAccountName(), + Namespace: etcd.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: etcd.GetRoleName(), + }, + } +} From 744f47bac723678d3f970f323fcbd2ef7b630407 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sun, 7 Jan 2024 14:22:25 +0530 Subject: [PATCH 062/235] add sample for rolebinding and memberlease, added tests for rolebinding and some tests for member lease --- .../operator/memberlease/memberlease_test.go | 98 +++++++++++++++++++ .../operator/rolebinding/rolebinding_test.go | 27 +++-- test/sample/memberlease.go | 27 +++++ test/sample/rolebinding.go | 1 + test/utils/fakeclient.go | 15 +++ 5 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 internal/operator/memberlease/memberlease_test.go create mode 100644 test/sample/memberlease.go diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go new file mode 100644 index 000000000..452e42dee --- /dev/null +++ b/internal/operator/memberlease/memberlease_test.go @@ -0,0 +1,98 @@ +package memberlease + +import ( + "context" + "errors" + "testing" + + "github.com/gardener/etcd-druid/internal/operator/resource" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) + +var ( + internalErr = errors.New("test internal error") +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + listErr := apierrors.NewInternalError(internalErr) + testCases := []struct { + name string + etcdReplicas int32 + numExistingLeases int + getErr *apierrors.StatusError + listErr *apierrors.StatusError + expectedErr *apierrors.StatusError + }{ + { + name: "lease exists for a single node etcd cluster", + etcdReplicas: 1, + numExistingLeases: 1, + }, + { + name: "all leases exist for a 3 node etcd cluster", + etcdReplicas: 3, + numExistingLeases: 3, + }, + { + name: "2 of 3 leases exist for a 3 node etcd cluster", + etcdReplicas: 3, + numExistingLeases: 2, + }, + { + name: "should return an empty slice when no member leases are found", + getErr: apierrors.NewNotFound(corev1.Resource("leases"), ""), + etcdReplicas: 3, + numExistingLeases: 0, + }, + { + name: "should return error when list fails", + listErr: listErr, + expectedErr: listErr, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() + fakeClientBuilder := testutils.NewFakeClientBuilder() + if tc.getErr != nil { + fakeClientBuilder.WithGetError(tc.getErr) + } + if tc.listErr != nil { + fakeClientBuilder.WithListError(tc.listErr) + } + if tc.numExistingLeases > 0 { + leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases) + g.Expect(err).ToNot(HaveOccurred()) + for _, lease := range leases { + fakeClientBuilder.WithObjects(lease) + } + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + memberLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + g.Expect(errors.Is(err, tc.expectedErr)).To(BeTrue()) + } else { + g.Expect(err).To(BeNil()) + } + g.Expect(len(memberLeaseNames), tc.numExistingLeases) + }) + } +} diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index 19757e28c..d8483ea23 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -33,6 +33,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + getErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string roleBindingExists bool @@ -41,25 +42,21 @@ func TestGetExistingResourceNames(t *testing.T) { expectedRoleBindingNames []string }{ { - "should return the existing role binding name", - true, - nil, - nil, - []string{etcd.GetRoleBindingName()}, + name: "should return the existing role binding name", + roleBindingExists: true, + expectedRoleBindingNames: []string{etcd.GetRoleBindingName()}, }, { - "should return empty slice when role binding is not found", - false, - apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleBindingName()), - nil, - []string{}, + name: "should return empty slice when role binding is not found", + roleBindingExists: false, + getErr: apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleBindingName()), + expectedRoleBindingNames: []string{}, }, { - "should return error when get fails", - true, - apierrors.NewInternalError(internalErr), - apierrors.NewInternalError(internalErr), - nil, + name: "should return error when get fails", + roleBindingExists: true, + getErr: getErr, + expectedErr: getErr, }, } diff --git a/test/sample/memberlease.go b/test/sample/memberlease.go new file mode 100644 index 000000000..2af33a463 --- /dev/null +++ b/test/sample/memberlease.go @@ -0,0 +1,27 @@ +package sample + +import ( + "errors" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func NewMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1.Lease, error) { + if numLeases > int(etcd.Spec.Replicas) { + return nil, errors.New("number of requested leases is greater than the etcd replicas") + } + memberLeaseNames := etcd.GetMemberLeaseNames() + leases := make([]*coordinationv1.Lease, 0, numLeases) + for i := 0; i < numLeases; i++ { + lease := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberLeaseNames[i], + Namespace: etcd.Namespace, + }, + } + leases = append(leases, lease) + } + return leases, nil +} diff --git a/test/sample/rolebinding.go b/test/sample/rolebinding.go index bafa3c3b9..4165fdced 100644 --- a/test/sample/rolebinding.go +++ b/test/sample/rolebinding.go @@ -6,6 +6,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// NewRoleBinding creates a new RoleBinding using the passed in etcd object. func NewRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/utils/fakeclient.go b/test/utils/fakeclient.go index f7433d551..3facfcc1b 100644 --- a/test/utils/fakeclient.go +++ b/test/utils/fakeclient.go @@ -13,6 +13,7 @@ import ( type FakeClientBuilder struct { clientBuilder *fakeclient.ClientBuilder getErr *apierrors.StatusError + listErr *apierrors.StatusError createErr *apierrors.StatusError patchErr *apierrors.StatusError deleteErr *apierrors.StatusError @@ -37,6 +38,11 @@ func (b *FakeClientBuilder) WithGetError(err *apierrors.StatusError) *FakeClient return b } +func (b *FakeClientBuilder) WithListError(err *apierrors.StatusError) *FakeClientBuilder { + b.listErr = err + return b +} + // WithCreateError sets the error that should be returned when a Create request is made on the fake client. func (b *FakeClientBuilder) WithCreateError(err *apierrors.StatusError) *FakeClientBuilder { b.createErr = err @@ -60,6 +66,7 @@ func (b *FakeClientBuilder) Build() client.WithWatch { return &fakeClient{ WithWatch: b.clientBuilder.Build(), getErr: b.getErr, + listErr: b.listErr, createErr: b.createErr, patchErr: b.patchErr, deleteErr: b.deleteErr, @@ -69,6 +76,7 @@ func (b *FakeClientBuilder) Build() client.WithWatch { type fakeClient struct { client.WithWatch getErr *apierrors.StatusError + listErr *apierrors.StatusError createErr *apierrors.StatusError patchErr *apierrors.StatusError deleteErr *apierrors.StatusError @@ -82,6 +90,13 @@ func (f *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.O return f.WithWatch.Get(ctx, key, obj, opts...) } +func (f *fakeClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if f.listErr != nil { + return f.listErr + } + return f.WithWatch.List(ctx, list, opts...) +} + // Delete overwrites the fake client Get implementation with a capability to return any configured error. func (f *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { if f.deleteErr != nil { From 9ca1e12d0ca3c783d83ed7bfa8c8753040cc50af Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 8 Jan 2024 12:27:45 +0530 Subject: [PATCH 063/235] corrected tests, get also now returns druiderr --- .../operator/clientservice/clientservice.go | 9 ++- .../clientservice/clientservice_test.go | 78 ++++++++----------- internal/operator/peerservice/peerservice.go | 15 +++- .../operator/peerservice/peerservice_test.go | 68 +++++++--------- internal/operator/role/role.go | 9 ++- internal/operator/role/role_test.go | 49 +++++------- internal/operator/rolebinding/rolebinding.go | 45 ++++++----- .../operator/rolebinding/rolebinding_test.go | 24 +++--- test/utils/errors.go | 18 +++++ 9 files changed, 158 insertions(+), 157 deletions(-) create mode 100644 test/utils/errors.go diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 69344cd24..f3d639d8c 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -23,6 +23,7 @@ const ( ) const ( + ErrGetClientService druidv1alpha1.ErrorCode = "ERR_GET_CLIENT_SERVICE" ErrDeleteClientService druidv1alpha1.ErrorCode = "ERR_DELETE_CLIENT_SERVICE" ErrSyncClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" ) @@ -34,11 +35,15 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) svc := &corev1.Service{} - if err := r.client.Get(ctx, getObjectKey(etcd), svc); err != nil { + svcObjectKey := getObjectKey(etcd) + if err := r.client.Get(ctx, svcObjectKey, svc); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrGetClientService, + "GetExistingResourceNames", + fmt.Sprintf("Error getting client service: %s for etcd: %v", svcObjectKey.Name, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, svc.Name) return resourceNames, nil diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index c0a105859..e1496e5bd 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -17,7 +17,6 @@ package clientservice import ( "context" "errors" - "fmt" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -47,34 +46,35 @@ const ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { internalErr := errors.New("test internal error") + getErr := apierrors.NewInternalError(internalErr) etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string svcExists bool getErr *apierrors.StatusError - expectedErr error + expectedErr *druiderr.DruidError expectedServiceNames []string }{ { - "should return the existing service name", - true, - nil, - nil, - []string{etcd.GetClientServiceName()}, + name: "should return the existing service name", + svcExists: true, + expectedServiceNames: []string{etcd.GetClientServiceName()}, }, { - "should return empty slice when service is not found", - false, - apierrors.NewNotFound(corev1.Resource("services"), etcd.GetClientServiceName()), - nil, - []string{}, + name: "should return empty slice when service is not found", + svcExists: false, + getErr: apierrors.NewNotFound(corev1.Resource("services"), etcd.GetClientServiceName()), + expectedServiceNames: []string{}, }, { - "should return error when get fails", - true, - apierrors.NewInternalError(internalErr), - apierrors.NewInternalError(internalErr), - nil, + name: "should return error when get fails", + svcExists: true, + getErr: getErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetClientService, + Cause: getErr, + Operation: "GetExistingResourceNames", + }, }, } @@ -94,7 +94,7 @@ func TestGetExistingResourceNames(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) } @@ -106,12 +106,13 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestClientServiceSync(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + getErr := apierrors.NewInternalError(errors.New("fake get error")) testCases := []struct { - name string - svcExists bool - setupFn func(eb *testsample.EtcdBuilder) - expectError *druiderr.DruidError - getErr *apierrors.StatusError + name string + svcExists bool + setupFn func(eb *testsample.EtcdBuilder) + expectedError *druiderr.DruidError + getErr *apierrors.StatusError }{ { name: "Create new service", @@ -133,13 +134,13 @@ func TestClientServiceSync(t *testing.T) { name: "With client error", svcExists: false, setupFn: nil, - expectError: &druiderr.DruidError{ + expectedError: &druiderr.DruidError{ Code: ErrSyncClientService, - Cause: fmt.Errorf("fake get error"), + Cause: getErr, Operation: "Sync", Message: "Error during create or update of client service", }, - getErr: apierrors.NewInternalError(errors.New("fake get error")), + getErr: getErr, }, } @@ -163,15 +164,8 @@ func TestClientServiceSync(t *testing.T) { operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) - if tc.expectError != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.getErr)).To(BeTrue()) - g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) - g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) - + if tc.expectedError != nil { + testutils.CheckDruidError(g, tc.expectedError, err) } else { g.Expect(err).NotTo(HaveOccurred()) service := &corev1.Service{ @@ -191,6 +185,7 @@ func TestClientServiceSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestClientServiceTriggerDelete(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + deleteErr := apierrors.NewInternalError(errors.New("fake delete error")) testCases := []struct { name string svcExists bool @@ -219,11 +214,11 @@ func TestClientServiceTriggerDelete(t *testing.T) { setupFn: nil, expectError: &druiderr.DruidError{ Code: ErrDeleteClientService, - Cause: errors.New("fake delete error"), + Cause: deleteErr, Operation: "TriggerDelete", Message: "Failed to delete client service", }, - deleteErr: apierrors.NewInternalError(errors.New("fake delete error")), + deleteErr: deleteErr, }, } g := NewWithT(t) @@ -247,14 +242,7 @@ func TestClientServiceTriggerDelete(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) if tc.expectError != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.deleteErr)).To(BeTrue()) - g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) - g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) - + testutils.CheckDruidError(g, tc.expectError, err) } else { g.Expect(err).NotTo(HaveOccurred()) serviceList := &corev1.List{} diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 74f9a5c99..44a74a24d 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -1,6 +1,8 @@ package peerservice import ( + "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" @@ -16,6 +18,7 @@ import ( const defaultServerPort = 2380 const ( + ErrGetPeerService druidv1alpha1.ErrorCode = "ERR_GET_PEER_SERVICE" ErrDeletePeerService druidv1alpha1.ErrorCode = "ERR_DELETE_PEER_SERVICE" ErrSyncPeerService druidv1alpha1.ErrorCode = "ERR_SYNC_PEER_SERVICE" ) @@ -26,12 +29,16 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) + svcObjectKey := getObjectKey(etcd) svc := &corev1.Service{} - if err := r.client.Get(ctx, getObjectKey(etcd), svc); err != nil { + if err := r.client.Get(ctx, svcObjectKey, svc); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrGetPeerService, + "GetExistingResourceNames", + fmt.Sprintf("Error getting peer service: %s for etcd: %v", svcObjectKey.Name, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, svc.Name) return resourceNames, nil @@ -56,7 +63,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncPeerService, "Sync", - "Error during create or update of peer service", + fmt.Sprintf("Error during create or update of peer service: %s for etcd: %v", svc.Name, etcd.GetNamespaceName()), ) } @@ -71,7 +78,7 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph err, ErrDeletePeerService, "TriggerDelete", - "Failed to delete peer service", + fmt.Sprintf("Failed to delete peer service: %s for etcd: %v", objectKey.Name, etcd.GetNamespaceName()), ) } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 273922522..bc1cd3523 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -17,7 +17,6 @@ package peerservice import ( "context" "errors" - "fmt" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -45,34 +44,35 @@ const ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { internalErr := errors.New("test internal error") + getInternalErr := apierrors.NewInternalError(internalErr) etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testcases := []struct { name string svcExists bool getErr *apierrors.StatusError - expectedErr error + expectedErr *druiderr.DruidError expectedServiceNames []string }{ { - "should return the existing service name", - true, - nil, - nil, - []string{etcd.GetPeerServiceName()}, + name: "should return the existing service name", + svcExists: true, + expectedServiceNames: []string{etcd.GetPeerServiceName()}, }, { - "should return empty slice when service is not found", - false, - apierrors.NewNotFound(corev1.Resource("services"), ""), - nil, - []string{}, + name: "should return empty slice when service is not found", + svcExists: false, + getErr: apierrors.NewNotFound(corev1.Resource("services"), ""), + expectedServiceNames: []string{}, }, { - "should return error when get fails", - true, - apierrors.NewInternalError(internalErr), - apierrors.NewInternalError(internalErr), - nil, + name: "should return error when get fails", + svcExists: true, + getErr: getInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetPeerService, + Cause: getInternalErr, + Operation: "GetExistingResourceNames", + }, }, } @@ -92,7 +92,7 @@ func TestGetExistingResourceNames(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) } @@ -104,12 +104,13 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestPeerServiceSync(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + getInternalErr := apierrors.NewInternalError(errors.New("fake get internal error")) testCases := []struct { name string svcExists bool setupFn func(eb *testsample.EtcdBuilder) - expectError *druiderr.DruidError getErr *apierrors.StatusError + expectError *druiderr.DruidError }{ { name: "Create new service", @@ -129,11 +130,11 @@ func TestPeerServiceSync(t *testing.T) { setupFn: nil, expectError: &druiderr.DruidError{ Code: ErrSyncPeerService, - Cause: fmt.Errorf("fake get error"), + Cause: getInternalErr, Operation: "Sync", Message: "Error during create or update of peer service", }, - getErr: apierrors.NewInternalError(errors.New("fake get error")), + getErr: getInternalErr, }, } @@ -158,14 +159,7 @@ func TestPeerServiceSync(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) if tc.expectError != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.getErr)).To(BeTrue()) - g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) - g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) - + testutils.CheckDruidError(g, tc.expectError, err) } else { g.Expect(err).NotTo(HaveOccurred()) service := &corev1.Service{ @@ -185,12 +179,13 @@ func TestPeerServiceSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestPeerServiceTriggerDelete(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + deleteInternalErr := apierrors.NewInternalError(errors.New("fake delete internal error")) testCases := []struct { name string svcExists bool setupFn func(eb *testsample.EtcdBuilder) - expectError *druiderr.DruidError deleteErr *apierrors.StatusError + expectError *druiderr.DruidError }{ { name: "Existing Service - Delete Operation", @@ -207,13 +202,13 @@ func TestPeerServiceTriggerDelete(t *testing.T) { name: "Client Error on Delete - Returns Error", svcExists: true, setupFn: nil, + deleteErr: deleteInternalErr, expectError: &druiderr.DruidError{ Code: ErrDeletePeerService, - Cause: errors.New("fake delete error"), + Cause: deleteInternalErr, Operation: "TriggerDelete", Message: "Failed to delete peer service", }, - deleteErr: apierrors.NewInternalError(errors.New("fake delete error")), }, } g := NewWithT(t) @@ -237,14 +232,7 @@ func TestPeerServiceTriggerDelete(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) if tc.expectError != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectError.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.deleteErr)).To(BeTrue()) - g.Expect(druidErr.Message).To(Equal(tc.expectError.Message)) - g.Expect(druidErr.Operation).To(Equal(tc.expectError.Operation)) - + testutils.CheckDruidError(g, tc.expectError, err) } else { g.Expect(err).NotTo(HaveOccurred()) serviceList := &corev1.List{} diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 1355fbd7f..8d634e768 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -14,6 +14,7 @@ import ( ) const ( + ErrGetRole druidv1alpha1.ErrorCode = "ERR_GET_ROLE" ErrSyncRole druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE" ErrDeleteRole druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE" ) @@ -24,12 +25,16 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) + roleObjectKey := getObjectKey(etcd) role := &rbacv1.Role{} - if err := r.client.Get(ctx, getObjectKey(etcd), role); err != nil { + if err := r.client.Get(ctx, roleObjectKey, role); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrGetRole, + "GetExistingResourceNames", + fmt.Sprintf("Error getting role: %s for etcd: %v", roleObjectKey.Name, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, role.Name) return resourceNames, nil diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 32a74d38d..70ece7219 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -32,33 +32,34 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + getInternalErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string roleExists bool getErr *apierrors.StatusError - expectedErr *apierrors.StatusError + expectedErr *druiderr.DruidError expectedRoleNames []string }{ { - "should return the existing role name", - true, - nil, - nil, - []string{etcd.GetRoleName()}, + name: "should return the existing role name", + roleExists: true, + expectedRoleNames: []string{etcd.GetRoleName()}, }, { - "should return empty slice when role is not found", - false, - apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleName()), - nil, - []string{}, + name: "should return empty slice when role is not found", + roleExists: false, + getErr: apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleName()), + expectedRoleNames: []string{}, }, { - "should return error when get fails", - true, - apierrors.NewInternalError(internalErr), - apierrors.NewInternalError(internalErr), - nil, + name: "should return error when get fails", + roleExists: true, + getErr: getInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetRole, + Cause: getInternalErr, + Operation: "GetExistingResourceNames", + }, }, } @@ -78,7 +79,7 @@ func TestGetExistingResourceNames(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) roleNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) } @@ -140,12 +141,7 @@ func TestSync(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) - g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).ToNot(HaveOccurred()) existingRole := &rbacv1.Role{ @@ -209,12 +205,7 @@ func TestTriggerDelete(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) - g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).NotTo(HaveOccurred()) existingRole := rbacv1.Role{} diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index 22a7c2a2b..e69ba11ed 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -14,6 +14,7 @@ import ( ) const ( + ErrGetRoleBinding druidv1alpha1.ErrorCode = "ERR_GET_ROLE_BINDING" ErrSyncRoleBinding druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE_BINDING" ErrDeleteRoleBinding druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE_BINDING" ) @@ -24,26 +25,32 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) + rbObjectKey := getObjectKey(etcd) rb := &rbacv1.RoleBinding{} - if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { + if err := r.client.Get(ctx, rbObjectKey, rb); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrGetRoleBinding, + "GetExistingResourceNames", + fmt.Sprintf("Error getting role-binding: %s for etcd: %v", rbObjectKey.Name, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, rb.Name) return resourceNames, nil } -func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { - rb := &rbacv1.RoleBinding{} - if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, err +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + ctx.Logger.Info("Triggering delete of role") + err := client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) + if err == nil { + ctx.Logger.Info("deleted", "resource", "role-binding", "name", etcd.GetRoleBindingName()) } - return true, nil + return druiderr.WrapError(err, + ErrDeleteRoleBinding, + "TriggerDelete", + fmt.Sprintf("Failed to delete role-binding: %s for etcd: %v", etcd.GetRoleBindingName(), etcd.GetNamespaceName()), + ) } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { @@ -75,17 +82,15 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ) } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of role") - err := client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) - if err == nil { - ctx.Logger.Info("deleted", "resource", "role-binding", "name", etcd.GetRoleBindingName()) +func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { + rb := &rbacv1.RoleBinding{} + if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err } - return druiderr.WrapError(err, - ErrDeleteRoleBinding, - "TriggerDelete", - fmt.Sprintf("Failed to delete role-binding: %s for etcd: %v", etcd.GetRoleBindingName(), etcd.GetNamespaceName()), - ) + return true, nil } func New(client client.Client) resource.Operator { diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index d8483ea23..f8978a8f7 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -38,7 +38,7 @@ func TestGetExistingResourceNames(t *testing.T) { name string roleBindingExists bool getErr *apierrors.StatusError - expectedErr *apierrors.StatusError + expectedErr *druiderr.DruidError expectedRoleBindingNames []string }{ { @@ -56,7 +56,11 @@ func TestGetExistingResourceNames(t *testing.T) { name: "should return error when get fails", roleBindingExists: true, getErr: getErr, - expectedErr: getErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetRoleBinding, + Cause: getErr, + Operation: "GetExistingResourceNames", + }, }, } @@ -76,7 +80,7 @@ func TestGetExistingResourceNames(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) roleBindingNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(errors.Is(err, tc.getErr)).To(BeTrue()) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) } @@ -138,12 +142,7 @@ func TestSync(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) - g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).ToNot(HaveOccurred()) existingRoleBinding := &rbacv1.RoleBinding{ @@ -207,12 +206,7 @@ func TestTriggerDelete(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(err).To(HaveOccurred()) - var druidErr *druiderr.DruidError - g.Expect(errors.As(err, &druidErr)).To(BeTrue()) - g.Expect(druidErr.Code).To(Equal(tc.expectedErr.Code)) - g.Expect(errors.Is(druidErr.Cause, tc.expectedErr.Cause)).To(BeTrue()) - g.Expect(druidErr.Operation).To(Equal(tc.expectedErr.Operation)) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).NotTo(HaveOccurred()) existingRoleBinding := rbacv1.RoleBinding{} diff --git a/test/utils/errors.go b/test/utils/errors.go new file mode 100644 index 000000000..783d208bc --- /dev/null +++ b/test/utils/errors.go @@ -0,0 +1,18 @@ +package utils + +import ( + "errors" + + druiderr "github.com/gardener/etcd-druid/internal/errors" + . "github.com/onsi/gomega" +) + +// CheckDruidError checks that an actual error is a DruidError and further checks its underline cause, error code and operation. +func CheckDruidError(g *WithT, expectedError *druiderr.DruidError, actualError error) { + g.Expect(actualError).To(HaveOccurred()) + var druidErr *druiderr.DruidError + g.Expect(errors.As(actualError, &druidErr)).To(BeTrue()) + g.Expect(druidErr.Code).To(Equal(expectedError.Code)) + g.Expect(errors.Is(druidErr.Cause, expectedError.Cause)).To(BeTrue()) + g.Expect(druidErr.Operation).To(Equal(expectedError.Operation)) +} From f6856486d1fe6808468ff996eadb34b4d25ca304 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 8 Jan 2024 15:50:47 +0530 Subject: [PATCH 064/235] fixed peer service tests --- internal/operator/peerservice/peerservice.go | 4 +- .../operator/peerservice/peerservice_test.go | 169 ++++++++++-------- 2 files changed, 98 insertions(+), 75 deletions(-) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 44a74a24d..27964f864 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -59,6 +59,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) }) if err == nil { ctx.Logger.Info("synced", "resource", "peer-service", "name", svc.Name, "result", result) + return nil } return druiderr.WrapError(err, ErrSyncPeerService, @@ -69,10 +70,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of client service") + ctx.Logger.Info("Triggering delete of peer service") err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPeerService(objectKey))) if err == nil { ctx.Logger.Info("deleted", "resource", "peer-service", "name", objectKey.Name) + return nil } return druiderr.WrapError( err, diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index bc1cd3523..b4c1e5fe3 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -33,6 +33,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -41,10 +42,13 @@ const ( testNs = "test-namespace" ) +var ( + internalErr = errors.New("fake get internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) +) + // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - internalErr := errors.New("test internal error") - getInternalErr := apierrors.NewInternalError(internalErr) etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testcases := []struct { name string @@ -67,10 +71,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", svcExists: true, - getErr: getInternalErr, + getErr: apiInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetPeerService, - Cause: getInternalErr, + Cause: apiInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -81,10 +85,7 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.svcExists { fakeClientBuilder.WithObjects(sample.NewPeerService(etcd)) } @@ -102,75 +103,100 @@ func TestGetExistingResourceNames(t *testing.T) { } // ----------------------------------- Sync ----------------------------------- -func TestPeerServiceSync(t *testing.T) { +func TestSyncWhenNoServiceExists(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) - getInternalErr := apierrors.NewInternalError(errors.New("fake get internal error")) testCases := []struct { - name string - svcExists bool - setupFn func(eb *testsample.EtcdBuilder) - getErr *apierrors.StatusError - expectError *druiderr.DruidError + name string + svcExists bool + createWithPort *int32 + createErr *apierrors.StatusError + expectedError *druiderr.DruidError }{ { - name: "Create new service", + name: "create peer service with default ports when none exists", svcExists: false, }, { - name: "Update existing service", - svcExists: true, - - setupFn: func(eb *testsample.EtcdBuilder) { - eb.WithBackupPort(nil) - }, + name: "create service when none exists with custom ports", + svcExists: false, + createWithPort: pointer.Int32(2222), }, { - name: "With client error", + name: "create fails when there is a create error", svcExists: false, - setupFn: nil, - expectError: &druiderr.DruidError{ + createErr: apiInternalErr, + expectedError: &druiderr.DruidError{ Code: ErrSyncPeerService, - Cause: getInternalErr, + Cause: apiInternalErr, Operation: "Sync", - Message: "Error during create or update of peer service", }, - getErr: getInternalErr, }, } - g := NewWithT(t) t.Parallel() - for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - if tc.setupFn != nil { - tc.setupFn(etcdBuilder) + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + if tc.createWithPort != nil { + etcdBuilder.WithEtcdServerPort(tc.createWithPort) } etcd := etcdBuilder.Build() - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } - if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewPeerService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) - } - cl := fakeClientBuilder.Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) - if tc.expectError != nil { - testutils.CheckDruidError(g, tc.expectError, err) + if tc.expectedError != nil { + testutils.CheckDruidError(g, tc.expectedError, err) } else { g.Expect(err).NotTo(HaveOccurred()) - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetPeerServiceName(), - Namespace: etcd.Namespace, - }, - } - err = cl.Get(opCtx, client.ObjectKeyFromObject(service), service) + checkPeerService(g, cl, etcd) + } + }) + } +} + +func TestSyncWhenServiceExists(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + svcExists bool + updateWithPort *int32 + patchErr *apierrors.StatusError + expectedError *druiderr.DruidError + }{ + { + name: "update peer service with new server port", + svcExists: true, + updateWithPort: pointer.Int32(2222), + }, + { + name: "update fails when there is a patch error", + svcExists: true, + updateWithPort: pointer.Int32(2222), + patchErr: apiInternalErr, + expectedError: &druiderr.DruidError{ + Code: ErrSyncPeerService, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + existingEtcd := etcdBuilder.Build() + cl := testutils.NewFakeClientBuilder().WithPatchError(tc.patchErr). + WithObjects(sample.NewPeerService(existingEtcd)). + Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + updatedEtcd := etcdBuilder.WithEtcdServerPort(tc.updateWithPort).Build() + err := operator.Sync(opCtx, updatedEtcd) + if tc.expectedError != nil { + testutils.CheckDruidError(g, tc.expectedError, err) + } else { g.Expect(err).NotTo(HaveOccurred()) - checkPeerService(g, service, etcd) + checkPeerService(g, cl, updatedEtcd) } }) } @@ -178,30 +204,25 @@ func TestPeerServiceSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestPeerServiceTriggerDelete(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() deleteInternalErr := apierrors.NewInternalError(errors.New("fake delete internal error")) testCases := []struct { name string svcExists bool - setupFn func(eb *testsample.EtcdBuilder) deleteErr *apierrors.StatusError expectError *druiderr.DruidError }{ { - name: "Existing Service - Delete Operation", + name: "no-op and no error if peer service not found", svcExists: false, }, { - name: "Service Not Found - No Operation", + name: "successfully deletes an existing peer service", svcExists: true, - setupFn: func(eb *testsample.EtcdBuilder) { - eb.WithEtcdServerPort(nil) - }, }, { - name: "Client Error on Delete - Returns Error", + name: "fails to delete on client delete error", svcExists: true, - setupFn: nil, deleteErr: deleteInternalErr, expectError: &druiderr.DruidError{ Code: ErrDeletePeerService, @@ -216,16 +237,9 @@ func TestPeerServiceTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - if tc.setupFn != nil { - tc.setupFn(etcdBuilder) - } - etcd := etcdBuilder.Build() - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.deleteErr != nil { - fakeClientBuilder.WithDeleteError(tc.deleteErr) - } + fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewPeerService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) + fakeClientBuilder.WithObjects(sample.NewPeerService(etcd)) } cl := fakeClientBuilder.Build() operator := New(cl) @@ -235,18 +249,18 @@ func TestPeerServiceTriggerDelete(t *testing.T) { testutils.CheckDruidError(g, tc.expectError, err) } else { g.Expect(err).NotTo(HaveOccurred()) - serviceList := &corev1.List{} - err = cl.List(opCtx, serviceList) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(serviceList.Items).To(BeEmpty()) - + svc := corev1.Service{} + err = cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace}, &svc) + g.Expect(err).ToNot(BeNil()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) } }) } } // ---------------------------- Helper Functions ----------------------------- -func checkPeerService(g *WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { +func checkPeerService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { + svc := getLatestPeerService(g, cl, etcd) peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) g.Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) g.Expect(svc.Labels).To(Equal(etcd.GetDefaultLabels())) @@ -264,3 +278,10 @@ func checkPeerService(g *WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { }), )) } + +func getLatestPeerService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) *corev1.Service { + svc := &corev1.Service{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace}, svc) + g.Expect(err).To(BeNil()) + return svc +} From df7e75b77f74d1e4b321f6764cdd68c8b3d33ed9 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 8 Jan 2024 16:25:13 +0530 Subject: [PATCH 065/235] fixed tests, added error handling to member lease --- .../operator/clientservice/clientservice.go | 19 ++- .../clientservice/clientservice_test.go | 156 +++++++++++------- internal/operator/memberlease/memberlease.go | 37 ++++- .../operator/peerservice/peerservice_test.go | 9 +- 4 files changed, 137 insertions(+), 84 deletions(-) diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index f3d639d8c..c39663397 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -64,6 +64,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) }) if err == nil { ctx.Logger.Info("synced", "resource", "client-service", "name", svc.Name, "result", result) + return nil } return druiderr.WrapError(err, ErrSyncClientService, @@ -75,16 +76,16 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of client service") - err := client.IgnoreNotFound(r.client.Delete(ctx, emptyClientService(objectKey))) - if err == nil { - ctx.Logger.Info("deleted", "resource", "client-service", "name", objectKey.Name) + if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyClientService(objectKey))); err != nil { + return druiderr.WrapError( + err, + ErrDeleteClientService, + "TriggerDelete", + "Failed to delete client service", + ) } - return druiderr.WrapError( - err, - ErrDeleteClientService, - "TriggerDelete", - "Failed to delete client service", - ) + ctx.Logger.Info("deleted", "resource", "client-service", "name", objectKey.Name) + return nil } func New(client client.Client) resource.Operator { diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index e1496e5bd..f4fd2acc8 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -21,6 +21,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" + "k8s.io/utils/pointer" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -43,10 +44,13 @@ const ( testNs = "test-namespace" ) +var ( + internalErr = errors.New("fake get internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) +) + // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - internalErr := errors.New("test internal error") - getErr := apierrors.NewInternalError(internalErr) etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string @@ -69,10 +73,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", svcExists: true, - getErr: getErr, + getErr: apiInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetClientService, - Cause: getErr, + Cause: apiInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -83,10 +87,7 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.svcExists { fakeClientBuilder.WithObjects(sample.NewClientService(etcd)) } @@ -104,63 +105,40 @@ func TestGetExistingResourceNames(t *testing.T) { } // ----------------------------------- Sync ----------------------------------- -func TestClientServiceSync(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) - getErr := apierrors.NewInternalError(errors.New("fake get error")) +func TestSyncWhenNoServiceExists(t *testing.T) { testCases := []struct { name string - svcExists bool - setupFn func(eb *testsample.EtcdBuilder) + clientPort *int32 + backupPort *int32 + peerPort *int32 + createErr *apierrors.StatusError expectedError *druiderr.DruidError - getErr *apierrors.StatusError }{ { - name: "Create new service", - svcExists: false, + name: "create client service with default ports", }, { - name: "Update existing service", - svcExists: true, - - setupFn: func(eb *testsample.EtcdBuilder) { - eb.WithEtcdClientPort(nil). - WithBackupPort(nil). - WithEtcdServerPort(nil). - WithEtcdClientServiceLabels(map[string]string{"testingKey": "testingValue"}). - WithEtcdClientServiceAnnotations(map[string]string{"testingAnnotationKey": "testingValue"}) - }, + name: "create client service with custom ports", + clientPort: pointer.Int32(2222), + backupPort: pointer.Int32(3333), + peerPort: pointer.Int32(4444), }, { - name: "With client error", - svcExists: false, - setupFn: nil, + name: "create fails when there is a create error", + createErr: apiInternalErr, expectedError: &druiderr.DruidError{ Code: ErrSyncClientService, - Cause: getErr, + Cause: apiInternalErr, Operation: "Sync", - Message: "Error during create or update of client service", }, - getErr: getErr, }, } - g := NewWithT(t) t.Parallel() - for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - if tc.setupFn != nil { - tc.setupFn(etcdBuilder) - } - etcd := etcdBuilder.Build() - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } - if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewClientService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) - } - cl := fakeClientBuilder.Build() + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + etcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) @@ -168,15 +146,54 @@ func TestClientServiceSync(t *testing.T) { testutils.CheckDruidError(g, tc.expectedError, err) } else { g.Expect(err).NotTo(HaveOccurred()) - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetClientServiceName(), - Namespace: etcd.Namespace, - }, - } - err = cl.Get(opCtx, client.ObjectKeyFromObject(service), service) + checkClientService(g, cl, etcd) + } + }) + } +} + +func TestSyncWhenServiceExists(t *testing.T) { + testCases := []struct { + name string + clientPort *int32 + backupPort *int32 + peerPort *int32 + patchErr *apierrors.StatusError + expectedError *druiderr.DruidError + }{ + { + name: "update peer service with new server port", + clientPort: pointer.Int32(2222), + peerPort: pointer.Int32(3333), + }, + { + name: "update fails when there is a patch error", + clientPort: pointer.Int32(2222), + patchErr: apiInternalErr, + expectedError: &druiderr.DruidError{ + Code: ErrSyncClientService, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + existingEtcd := buildEtcd(nil, nil, nil) + cl := testutils.NewFakeClientBuilder().WithPatchError(tc.patchErr). + WithObjects(sample.NewClientService(existingEtcd)). + Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + updatedEtcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) + err := operator.Sync(opCtx, updatedEtcd) + if tc.expectedError != nil { + testutils.CheckDruidError(g, tc.expectedError, err) + } else { g.Expect(err).NotTo(HaveOccurred()) - checkClientService(g, service, etcd) + checkClientService(g, cl, updatedEtcd) } }) } @@ -185,7 +202,6 @@ func TestClientServiceSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestClientServiceTriggerDelete(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) - deleteErr := apierrors.NewInternalError(errors.New("fake delete error")) testCases := []struct { name string svcExists bool @@ -214,11 +230,10 @@ func TestClientServiceTriggerDelete(t *testing.T) { setupFn: nil, expectError: &druiderr.DruidError{ Code: ErrDeleteClientService, - Cause: deleteErr, + Cause: apiInternalErr, Operation: "TriggerDelete", - Message: "Failed to delete client service", }, - deleteErr: deleteErr, + deleteErr: apiInternalErr, }, } g := NewWithT(t) @@ -255,7 +270,23 @@ func TestClientServiceTriggerDelete(t *testing.T) { } } -func checkClientService(g *WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) { +// ---------------------------- Helper Functions ----------------------------- +func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + if clientPort != nil { + etcdBuilder.WithEtcdClientPort(clientPort) + } + if peerPort != nil { + etcdBuilder.WithEtcdServerPort(peerPort) + } + if backupPort != nil { + etcdBuilder.WithBackupPort(backupPort) + } + return etcdBuilder.Build() +} + +func checkClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { + svc := getLatestClientService(g, cl, etcd) clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) @@ -293,3 +324,10 @@ func checkClientService(g *WithT, svc *corev1.Service, etcd *druidv1alpha1.Etcd) }), )) } + +func getLatestClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) *corev1.Service { + svc := &corev1.Service{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}, svc) + g.Expect(err).To(BeNil()) + return svc +} diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index dfc9f0424..738907601 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -1,8 +1,11 @@ package memberlease import ( + "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/hashicorp/go-multierror" @@ -14,6 +17,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrListMemberLease druidv1alpha1.ErrorCode = "ERR_LIST_MEMBER_LEASE" + ErrDeleteMemberLease druidv1alpha1.ErrorCode = "ERR_DELETE_MEMBER_LEASE" + ErrSyncMemberLease druidv1alpha1.ErrorCode = "ERR_SYNC_MEMBER_LEASE" +) + const purpose = "etcd-member-lease" type _resource struct { @@ -28,7 +37,10 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * client.InNamespace(etcd.Namespace), client.MatchingLabels(getLabels(etcd))) if err != nil { - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrListMemberLease, + "GetExistingResourceNames", + fmt.Sprintf("Error listing member leases for etcd: %v", etcd.GetNamespaceName())) } for _, lease := range leaseList.Items { resourceNames = append(resourceNames, lease.Name) @@ -50,15 +62,12 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) }, } } - if errorList := utils.RunConcurrently(ctx, createTasks); len(errorList) > 0 { for _, err := range errorList { errs = multierror.Append(errs, err) } - return errs } - - return nil + return errs } func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { @@ -69,17 +78,29 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a return nil }) if err != nil { - return err + return druiderr.WrapError(err, + ErrSyncMemberLease, + "Sync", + fmt.Sprintf("Error syncing member lease: %s for etcd: %v", objKey.Name, etcd.GetNamespaceName())) } - ctx.Logger.Info("triggered creation of member lease", "lease", objKey, "operationResult", opResult) + ctx.Logger.Info("triggered create or update of member lease", "lease", objKey, "operationResult", opResult) return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return r.client.DeleteAllOf(ctx, + ctx.Logger.Info("Triggering delete of member leases") + err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(etcd.Namespace), client.MatchingLabels(etcd.GetDefaultLabels())) + if err == nil { + ctx.Logger.Info("deleted", "resource", "member-leases") + return nil + } + return druiderr.WrapError(err, + ErrDeleteMemberLease, + "TriggerDelete", + fmt.Sprintf("Failed to delete member leases for etcd: %v", etcd.GetNamespaceName())) } func New(client client.Client) resource.Operator { diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index b4c1e5fe3..66987b175 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -107,23 +107,19 @@ func TestSyncWhenNoServiceExists(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string - svcExists bool createWithPort *int32 createErr *apierrors.StatusError expectedError *druiderr.DruidError }{ { - name: "create peer service with default ports when none exists", - svcExists: false, + name: "create peer service with default ports when none exists", }, { name: "create service when none exists with custom ports", - svcExists: false, createWithPort: pointer.Int32(2222), }, { name: "create fails when there is a create error", - svcExists: false, createErr: apiInternalErr, expectedError: &druiderr.DruidError{ Code: ErrSyncPeerService, @@ -158,19 +154,16 @@ func TestSyncWhenServiceExists(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string - svcExists bool updateWithPort *int32 patchErr *apierrors.StatusError expectedError *druiderr.DruidError }{ { name: "update peer service with new server port", - svcExists: true, updateWithPort: pointer.Int32(2222), }, { name: "update fails when there is a patch error", - svcExists: true, updateWithPort: pointer.Int32(2222), patchErr: apiInternalErr, expectedError: &druiderr.DruidError{ From 8266dce3624ff3ec58815a17f3d754f90c68b737 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 9 Jan 2024 13:32:25 +0530 Subject: [PATCH 066/235] added member lease test and minor corrections to clientservice test --- .../clientservice/clientservice_test.go | 23 +- .../operator/memberlease/memberlease_test.go | 238 ++++++++++++++++-- test/sample/memberlease.go | 9 +- test/utils/fakeclient.go | 49 ++-- 4 files changed, 265 insertions(+), 54 deletions(-) diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index f4fd2acc8..626edb49c 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -200,34 +200,24 @@ func TestSyncWhenServiceExists(t *testing.T) { } // ----------------------------- TriggerDelete ------------------------------- -func TestClientServiceTriggerDelete(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) +func TestTriggerDelete(t *testing.T) { testCases := []struct { name string svcExists bool - setupFn func(eb *testsample.EtcdBuilder) expectError *druiderr.DruidError deleteErr *apierrors.StatusError }{ { - name: "Existing Service - Delete Operation", + name: "no-op when client service does not exist", svcExists: false, }, { - name: "Service Not Found - No Operation", + name: "successfully delete client service", svcExists: true, - setupFn: func(eb *testsample.EtcdBuilder) { - eb.WithEtcdClientPort(nil). - WithBackupPort(nil). - WithEtcdServerPort(nil). - WithEtcdClientServiceLabels(map[string]string{"testingKey": "testingValue"}). - WithEtcdClientServiceAnnotations(map[string]string{"testingAnnotationKey": "testingValue"}) - }, }, { - name: "Client Error on Delete - Returns Error", + name: "returns error when client delete fails", svcExists: true, - setupFn: nil, expectError: &druiderr.DruidError{ Code: ErrDeleteClientService, Cause: apiInternalErr, @@ -241,10 +231,7 @@ func TestClientServiceTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - if tc.setupFn != nil { - tc.setupFn(etcdBuilder) - } - etcd := etcdBuilder.Build() + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() fakeClientBuilder := testutils.NewFakeClientBuilder() if tc.deleteErr != nil { fakeClientBuilder.WithDeleteError(tc.deleteErr) diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 452e42dee..30c675a9e 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -3,16 +3,26 @@ package memberlease import ( "context" "errors" + "fmt" "testing" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -21,20 +31,20 @@ const ( ) var ( - internalErr = errors.New("test internal error") + internalErr = errors.New("test internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) ) // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) - listErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string etcdReplicas int32 numExistingLeases int getErr *apierrors.StatusError listErr *apierrors.StatusError - expectedErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { name: "lease exists for a single node etcd cluster", @@ -58,9 +68,13 @@ func TestGetExistingResourceNames(t *testing.T) { numExistingLeases: 0, }, { - name: "should return error when list fails", - listErr: listErr, - expectedErr: listErr, + name: "should return error when list fails", + listErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrListMemberLease, + Cause: apiInternalErr, + Operation: "GetExistingResourceNames", + }, }, } @@ -70,15 +84,11 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } - if tc.listErr != nil { - fakeClientBuilder.WithListError(tc.listErr) - } + fakeClientBuilder := testutils.NewFakeClientBuilder(). + WithGetError(tc.getErr). + WithListError(tc.listErr) if tc.numExistingLeases > 0 { - leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases) + leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { fakeClientBuilder.WithObjects(lease) @@ -88,11 +98,207 @@ func TestGetExistingResourceNames(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) memberLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { - g.Expect(errors.Is(err, tc.expectedErr)).To(BeTrue()) + testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) + g.Expect(len(memberLeaseNames), tc.numExistingLeases) + } + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSync(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + etcdReplicas int32 // original replicas + deltaEtcdReplicas int32 // change in etcd replicas if any + numExistingLeases int + createErr *apierrors.StatusError + getErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "create member leases for a single node etcd cluster", + etcdReplicas: 1, + numExistingLeases: 0, + }, + { + name: "creates member leases when etcd replicas is changes from 1 -> 3", + etcdReplicas: 1, + numExistingLeases: 1, + deltaEtcdReplicas: 2, + }, + { + name: "should return error when client create fails", + etcdReplicas: 3, + numExistingLeases: 0, + createErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncMemberLease, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + { + name: "should return error when client get fails", + etcdReplicas: 3, + numExistingLeases: 0, + getErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncMemberLease, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // *************** set up existing environment ***************** + etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() + fakeClientBuilder := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).WithGetError(tc.getErr) + if tc.numExistingLeases > 0 { + leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) + g.Expect(err).ToNot(HaveOccurred()) + for _, lease := range leases { + fakeClientBuilder.WithObjects(lease) + } + } + cl := fakeClientBuilder.Build() + + // ***************** Setup updated etcd to be passed to the Sync method ***************** + var updatedEtcd *druidv1alpha1.Etcd + // Currently we only support up-scaling. If and when down-scaling is supported, this condition will have to change. + if tc.deltaEtcdReplicas > 0 { + updatedEtcd = etcdBuilder.WithReplicas(tc.etcdReplicas + tc.deltaEtcdReplicas).Build() + } else { + updatedEtcd = etcd + } + // ***************** Setup operator and test ***************** + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, updatedEtcd) + memberLeasesPostSync := getLatestMemberLeases(g, cl, updatedEtcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + g.Expect(memberLeasesPostSync).Should(HaveLen(tc.numExistingLeases)) + } else { + g.Expect(memberLeasesPostSync).To(ConsistOf(memberLeases(updatedEtcd.Name, updatedEtcd.UID, updatedEtcd.Spec.Replicas))) } - g.Expect(len(memberLeaseNames), tc.numExistingLeases) }) } } + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + testCases := []struct { + name string + etcdReplicas int32 // original replicas + numExistingLeases int + deleteAllOfErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "no-op when no member lease exists", + etcdReplicas: 3, + numExistingLeases: 0, + }, + { + name: "successfully deletes all member leases", + etcdReplicas: 3, + numExistingLeases: 3, + }, + { + name: "successfully deletes remainder member leases", + etcdReplicas: 3, + numExistingLeases: 1, + }, + { + name: "returns error when client delete fails", + etcdReplicas: 3, + numExistingLeases: 3, + deleteAllOfErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeleteMemberLease, + Cause: apiInternalErr, + Operation: "TriggerDelete", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // *************** set up existing environment ***************** + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).WithReplicas(tc.etcdReplicas).Build() + fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllOfErr) + if tc.numExistingLeases > 0 { + leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) + g.Expect(err).ToNot(HaveOccurred()) + for _, lease := range leases { + fakeClientBuilder.WithObjects(lease) + } + } + cl := fakeClientBuilder.Build() + // ***************** Setup operator and test ***************** + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.TriggerDelete(opCtx, etcd) + memberLeasesPostDelete := getLatestMemberLeases(g, cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + g.Expect(memberLeasesPostDelete).Should(HaveLen(tc.numExistingLeases)) + } else { + g.Expect(memberLeasesPostDelete).Should(HaveLen(0)) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- + +func getAdditionalMemberLeaseLabels(etcdName string) map[string]string { + return map[string]string{ + common.GardenerOwnedBy: etcdName, + v1beta1constants.GardenerPurpose: purpose, + } +} + +func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { + leases := &coordinationv1.LeaseList{} + g.Expect(cl.List(context.Background(), leases, client.InNamespace(etcd.Namespace), client.MatchingLabels(map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-member-lease", + }))).To(Succeed()) + return leases.Items +} + +func memberLeases(etcdName string, etcdUID types.UID, replicas int32) []interface{} { + var elements []interface{} + for i := 0; i < int(replicas); i++ { + leaseName := fmt.Sprintf("%s-%d", etcdName, i) + elements = append(elements, matchLeaseElement(leaseName, etcdName, etcdUID)) + } + return elements +} + +func matchLeaseElement(leaseName, etcdName string, etcdUID types.UID) gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(leaseName), + "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ + "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), + "Kind": Equal("Etcd"), + "Name": Equal(etcdName), + "UID": Equal(etcdUID), + "Controller": PointTo(BeTrue()), + "BlockOwnerDeletion": PointTo(BeTrue()), + })), + }), + }) +} diff --git a/test/sample/memberlease.go b/test/sample/memberlease.go index 2af33a463..60d54e1e0 100644 --- a/test/sample/memberlease.go +++ b/test/sample/memberlease.go @@ -4,11 +4,12 @@ import ( "errors" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + testutils "github.com/gardener/etcd-druid/test/utils" coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func NewMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1.Lease, error) { +func NewMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int, additionalLabels map[string]string) ([]*coordinationv1.Lease, error) { if numLeases > int(etcd.Spec.Replicas) { return nil, errors.New("number of requested leases is greater than the etcd replicas") } @@ -17,8 +18,10 @@ func NewMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 for i := 0; i < numLeases; i++ { lease := &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: memberLeaseNames[i], - Namespace: etcd.Namespace, + Name: memberLeaseNames[i], + Namespace: etcd.Namespace, + Labels: testutils.MergeMaps[string, string](etcd.GetDefaultLabels(), additionalLabels), + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, }, } leases = append(leases, lease) diff --git a/test/utils/fakeclient.go b/test/utils/fakeclient.go index 3facfcc1b..b170e73ce 100644 --- a/test/utils/fakeclient.go +++ b/test/utils/fakeclient.go @@ -11,12 +11,13 @@ import ( // FakeClientBuilder builds a fake client with initial set of objects and additionally // provides an ability to set custom errors for operations supported on the client. type FakeClientBuilder struct { - clientBuilder *fakeclient.ClientBuilder - getErr *apierrors.StatusError - listErr *apierrors.StatusError - createErr *apierrors.StatusError - patchErr *apierrors.StatusError - deleteErr *apierrors.StatusError + clientBuilder *fakeclient.ClientBuilder + getErr *apierrors.StatusError + listErr *apierrors.StatusError + createErr *apierrors.StatusError + patchErr *apierrors.StatusError + deleteErr *apierrors.StatusError + deleteAllOfErr *apierrors.StatusError } // NewFakeClientBuilder creates a new FakeClientBuilder. @@ -61,25 +62,32 @@ func (b *FakeClientBuilder) WithDeleteError(err *apierrors.StatusError) *FakeCli return b } +func (b *FakeClientBuilder) WithDeleteAllOfError(err *apierrors.StatusError) *FakeClientBuilder { + b.deleteAllOfErr = err + return b +} + // Build returns an instance of client.WithWatch which has capability to return the configured errors for operations. func (b *FakeClientBuilder) Build() client.WithWatch { return &fakeClient{ - WithWatch: b.clientBuilder.Build(), - getErr: b.getErr, - listErr: b.listErr, - createErr: b.createErr, - patchErr: b.patchErr, - deleteErr: b.deleteErr, + WithWatch: b.clientBuilder.Build(), + getErr: b.getErr, + listErr: b.listErr, + createErr: b.createErr, + patchErr: b.patchErr, + deleteErr: b.deleteErr, + deleteAllOfErr: b.deleteAllOfErr, } } type fakeClient struct { client.WithWatch - getErr *apierrors.StatusError - listErr *apierrors.StatusError - createErr *apierrors.StatusError - patchErr *apierrors.StatusError - deleteErr *apierrors.StatusError + getErr *apierrors.StatusError + listErr *apierrors.StatusError + createErr *apierrors.StatusError + patchErr *apierrors.StatusError + deleteErr *apierrors.StatusError + deleteAllOfErr *apierrors.StatusError } // Get overwrites the fake client Get implementation with a capability to return any configured error. @@ -105,6 +113,13 @@ func (f *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie return f.WithWatch.Delete(ctx, obj, opts...) } +func (f *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if f.deleteAllOfErr != nil { + return f.deleteAllOfErr + } + return f.WithWatch.DeleteAllOf(ctx, obj, opts...) +} + func (f *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if f.patchErr != nil { return f.patchErr From 28e825effad03d7650b54d40bef263436f176318 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 9 Jan 2024 17:15:47 +0530 Subject: [PATCH 067/235] added service account test --- .../clientservice/clientservice_test.go | 28 +-- .../operator/memberlease/memberlease_test.go | 4 +- .../operator/serviceaccount/serviceaccount.go | 38 +++- .../serviceaccount/serviceaccount_test.go | 212 +++++++++++++++++- test/sample/serviceaccount.go | 21 ++ 5 files changed, 279 insertions(+), 24 deletions(-) create mode 100644 test/sample/serviceaccount.go diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 626edb49c..eb229f81b 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -232,26 +232,22 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.deleteErr != nil { - fakeClientBuilder.WithDeleteError(tc.deleteErr) - } + fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.svcExists { fakeClientBuilder.WithObjects(sample.NewClientService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) } cl := fakeClientBuilder.Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.TriggerDelete(opCtx, etcd) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + latestClientService, getErr := getLatestClientService(cl, etcd) if tc.expectError != nil { - testutils.CheckDruidError(g, tc.expectError, err) + testutils.CheckDruidError(g, tc.expectError, triggerDeleteErr) + g.Expect(getErr).To(BeNil()) + g.Expect(latestClientService).ToNot(BeNil()) } else { - g.Expect(err).NotTo(HaveOccurred()) - serviceList := &corev1.List{} - err = cl.List(opCtx, serviceList) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(serviceList.Items).To(BeEmpty()) - + g.Expect(triggerDeleteErr).NotTo(HaveOccurred()) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } }) } @@ -273,7 +269,8 @@ func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { } func checkClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { - svc := getLatestClientService(g, cl, etcd) + svc, err := getLatestClientService(cl, etcd) + g.Expect(err).To(BeNil()) clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) @@ -312,9 +309,8 @@ func checkClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { )) } -func getLatestClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) *corev1.Service { +func getLatestClientService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { svc := &corev1.Service{} err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}, svc) - g.Expect(err).To(BeNil()) - return svc + return svc, err } diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 30c675a9e..80dbd7654 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -88,7 +88,7 @@ func TestGetExistingResourceNames(t *testing.T) { WithGetError(tc.getErr). WithListError(tc.listErr) if tc.numExistingLeases > 0 { - leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) + leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Namespace)) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { fakeClientBuilder.WithObjects(lease) @@ -248,6 +248,8 @@ func TestTriggerDelete(t *testing.T) { // ***************** Setup operator and test ***************** operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + memberLeasesBeforeDelete := getLatestMemberLeases(g, cl, etcd) + fmt.Println(memberLeasesBeforeDelete) err := operator.TriggerDelete(opCtx, etcd) memberLeasesPostDelete := getLatestMemberLeases(g, cl, etcd) if tc.expectedErr != nil { diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 1ccd88672..1f6ed5315 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -1,7 +1,10 @@ package serviceaccount import ( + "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -11,6 +14,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrGetServiceAccount druidv1alpha1.ErrorCode = "ERR_GET_SERVICE_ACCOUNT" + ErrDeleteServiceAccount druidv1alpha1.ErrorCode = "ERR_DELETE_SERVICE_ACCOUNT" + ErrSyncServiceAccount druidv1alpha1.ErrorCode = "ERR_SYNC_SERVICE_ACCOUNT" +) + type _resource struct { client client.Client disableAutoMount bool @@ -19,11 +28,15 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) sa := &corev1.ServiceAccount{} - if err := r.client.Get(ctx, getObjectKey(etcd), sa); err != nil { + saObjectKey := getObjectKey(etcd) + if err := r.client.Get(ctx, saObjectKey, sa); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrGetServiceAccount, + "GetExistingResourceNames", + fmt.Sprintf("Error getting service account: %s for etcd: %v", saObjectKey.Name, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, sa.Name) return resourceNames, nil @@ -37,12 +50,27 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) sa.AutomountServiceAccountToken = pointer.Bool(!r.disableAutoMount) return nil }) - ctx.Logger.Info("TriggerCreateOrUpdate operation result", "result", opResult) - return err + if err == nil { + ctx.Logger.Info("synced", "resource", "service-account", "name", sa.Name, "result", opResult) + return nil + } + return druiderr.WrapError(err, + ErrSyncServiceAccount, + "Sync", + fmt.Sprintf("Error during create or update of service account: %s for etcd: %v", sa.Name, etcd.GetNamespaceName())) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return client.IgnoreNotFound(r.client.Delete(ctx, emptyServiceAccount(getObjectKey(etcd)))) + ctx.Logger.Info("Triggering delete of service account") + saObjectKey := getObjectKey(etcd) + if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyServiceAccount(saObjectKey))); err != nil { + return druiderr.WrapError(err, + ErrDeleteServiceAccount, + "TriggerDelete", + fmt.Sprintf("Failed to delete service account: %s for etcd: %v", saObjectKey.Name, etcd.GetNamespaceName())) + } + ctx.Logger.Info("deleted", "resource", "service-account", "name", saObjectKey.Name) + return nil } func New(client client.Client, disableAutomount bool) resource.Operator { diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index de32bbbf3..3d68aeb11 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -1,12 +1,220 @@ package serviceaccount import ( + "context" + "errors" "testing" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + gomegatypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" +) + +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) + +var ( + internalErr = errors.New("fake get internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) ) -func DummyTest(t *testing.T) { +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testCases := []struct { + name string + saExists bool + getErr *apierrors.StatusError + expectedErr *druiderr.DruidError + expectedSANames []string + }{ + { + name: "should return empty slice, when no service account exists", + saExists: false, + getErr: apierrors.NewNotFound(corev1.Resource("serviceaccounts"), etcd.GetServiceAccountName()), + expectedSANames: []string{}, + }, + { + name: "should return existing service account name", + saExists: true, + expectedSANames: []string{etcd.GetServiceAccountName()}, + }, + { + name: "should return err when client get fails", + saExists: true, + getErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetServiceAccount, + Cause: apiInternalErr, + Operation: "GetExistingResourceNames", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + if tc.saExists { + fakeClientBuilder.WithObjects(testsample.NewServiceAccount(etcd, false)) + } + cl := fakeClientBuilder.Build() + operator := New(cl, true) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + saNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + } else { + if tc.saExists { + existingSA, err := getLatestServiceAccount(cl, etcd) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(saNames).To(HaveLen(1)) + g.Expect(saNames[0]).To(Equal(existingSA.Name)) + } else { + g.Expect(saNames).To(HaveLen(0)) + } + } + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSync(t *testing.T) { + testCases := []struct { + name string + disableAutoMount bool + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "create service account when none exists", + }, + { + name: "create service account with disabled auto mount", + disableAutoMount: true, + }, + { + name: "should return err when client create fails", + createErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncServiceAccount, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + operator := New(cl, tc.disableAutoMount) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + syncErr := operator.Sync(opCtx, etcd) + latestSA, getErr := getLatestServiceAccount(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } else { + g.Expect(getErr).To(BeNil()) + g.Expect(*latestSA).To(matchServiceAccount(etcd.GetServiceAccountName(), etcd.Namespace, etcd.Name, etcd.UID, tc.disableAutoMount)) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + testCases := []struct { + name string + saExists bool + deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "no-op when service account does not exist", + saExists: false, + }, + { + name: "successfully delete service account", + saExists: true, + }, + { + name: "returns error when client delete fails", + saExists: true, + deleteErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeleteServiceAccount, + Cause: apiInternalErr, + Operation: "TriggerDelete", + }, + }, + } + g := NewWithT(t) - g.Expect("bingo").To(Equal("bingo")) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) + if tc.saExists { + fakeClientBuilder.WithObjects(testsample.NewServiceAccount(etcd, false)) + } + cl := fakeClientBuilder.Build() + operator := New(cl, false) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + latestSA, getErr := getLatestServiceAccount(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) + g.Expect(getErr).To(BeNil()) + g.Expect(latestSA).ToNot(BeNil()) + } else { + g.Expect(triggerDeleteErr).To(BeNil()) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- +func getLatestServiceAccount(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.ServiceAccount, error) { + sa := &corev1.ServiceAccount{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace}, sa) + return sa, err +} + +func matchServiceAccount(saName, saNamespace, etcdName string, etcdUID types.UID, disableAutomount bool) gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(saName), + "Namespace": Equal(saNamespace), + "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ + "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), + "Kind": Equal("Etcd"), + "Name": Equal(etcdName), + "UID": Equal(etcdUID), + "Controller": PointTo(BeTrue()), + "BlockOwnerDeletion": PointTo(BeTrue()), + })), + }), + "AutomountServiceAccountToken": PointTo(Equal(!disableAutomount)), + }) } diff --git a/test/sample/serviceaccount.go b/test/sample/serviceaccount.go new file mode 100644 index 000000000..aaa683f73 --- /dev/null +++ b/test/sample/serviceaccount.go @@ -0,0 +1,21 @@ +package sample + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +// NewServiceAccount creates a new sample ServiceAccount using the passed in etcd. +func NewServiceAccount(etcd *druidv1alpha1.Etcd, disableAutoMount bool) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetServiceAccountName(), + Namespace: etcd.Namespace, + Labels: etcd.GetDefaultLabels(), + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + AutomountServiceAccountToken: pointer.Bool(!disableAutoMount), + } +} From 8ab20b47f806e6983a4636b4206e078a82946563 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 10 Jan 2024 13:23:01 +0530 Subject: [PATCH 068/235] added pdb operator unit test --- .../operator/peerservice/peerservice_test.go | 5 +- .../poddisruptionbudget.go | 34 ++- .../poddisruptionbudget_test.go | 272 ++++++++++++++++++ test/sample/pdb.go | 43 +++ 4 files changed, 347 insertions(+), 7 deletions(-) create mode 100644 internal/operator/poddistruptionbudget/poddisruptionbudget_test.go create mode 100644 test/sample/pdb.go diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 66987b175..e2c3a3303 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -119,7 +119,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { createWithPort: pointer.Int32(2222), }, { - name: "create fails when there is a create error", + name: "returns error when client create fails", createErr: apiInternalErr, expectedError: &druiderr.DruidError{ Code: ErrSyncPeerService, @@ -214,14 +214,13 @@ func TestPeerServiceTriggerDelete(t *testing.T) { svcExists: true, }, { - name: "fails to delete on client delete error", + name: "returns error when client delete fails", svcExists: true, deleteErr: deleteInternalErr, expectError: &druiderr.DruidError{ Code: ErrDeletePeerService, Cause: deleteInternalErr, Operation: "TriggerDelete", - Message: "Failed to delete peer service", }, }, } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index cb80ada44..6b3c7813b 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -5,6 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/gardener/pkg/controllerutils" policyv1 "k8s.io/api/policy/v1" @@ -14,6 +15,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrGetPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_GET_POD_DISRUPTION_BUDGET" + ErrDeletePodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_DELETE_POD_DISRUPTION_BUDGET" + ErrSyncPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_SYNC_POD_DISRUPTION_BUDGET" +) + type _resource struct { client client.Client } @@ -25,7 +32,10 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return resourceNames, druiderr.WrapError(err, + ErrGetPodDisruptionBudget, + "GetExistingResourceNames", + fmt.Sprintf("Error getting PDB: %s for etcd: %v", pdb.Name, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, pdb.Name) return resourceNames, nil @@ -33,7 +43,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { pdb := emptyPodDisruptionBudget(getObjectKey(etcd)) - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { + result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { pdb.Labels = etcd.GetDefaultLabels() pdb.Annotations = getAnnotations(etcd) pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} @@ -46,11 +56,27 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } return nil }) - return err + if err == nil { + ctx.Logger.Info("synced", "resource", "pod-disruption-budget", "name", pdb.Name, "result", result) + return nil + } + return druiderr.WrapError(err, + ErrSyncPodDisruptionBudget, + "Sync", + fmt.Sprintf("Error during create or update of PDB: %s for etcd: %v", pdb.Name, etcd.GetNamespaceName())) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return r.client.Delete(ctx, emptyPodDisruptionBudget(getObjectKey(etcd))) + ctx.Logger.Info("Triggering delete of PDB") + pdbObjectKey := getObjectKey(etcd) + if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPodDisruptionBudget(pdbObjectKey))); err != nil { + return druiderr.WrapError(err, + ErrDeletePodDisruptionBudget, + "TriggerDelete", + fmt.Sprintf("Failed to delete PDB: %s for etcd: %v", pdbObjectKey.Name, etcd.GetNamespaceName())) + } + ctx.Logger.Info("deleted", "resource", "pod-disruption-budget", "name", pdbObjectKey.Name) + return nil } func New(client client.Client) resource.Operator { diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go new file mode 100644 index 000000000..19c419b95 --- /dev/null +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -0,0 +1,272 @@ +package poddistruptionbudget + +import ( + "context" + "errors" + "fmt" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/gomega" +) + +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) + +var ( + internalErr = errors.New("fake get internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testCases := []struct { + name string + pdbExists bool + getErr *apierrors.StatusError + expectedErr *druiderr.DruidError + expectedPDBNames []string + }{ + { + name: "should return the existing PDB", + pdbExists: true, + expectedPDBNames: []string{etcd.Name}, + }, + { + name: "should return empty slice when PDB is not found", + pdbExists: false, + getErr: apierrors.NewNotFound(corev1.Resource("services"), etcd.GetClientServiceName()), + expectedPDBNames: []string{}, + }, + { + name: "should return error when get fails", + pdbExists: true, + getErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetPodDisruptionBudget, + Cause: apiInternalErr, + Operation: "GetExistingResourceNames", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + if tc.pdbExists { + fakeClientBuilder.WithObjects(testsample.NewPodDisruptionBudget(etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + pdbNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + } else { + g.Expect(err).To(BeNil()) + } + g.Expect(pdbNames, tc.expectedPDBNames) + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSyncWhenNoPDBExists(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + etcdReplicas int32 + createErr *apierrors.StatusError + expectedPDBMinAvailable int32 + expectedErr *druiderr.DruidError + }{ + { + name: "create PDB for single node etcd cluster when none exists", + etcdReplicas: 1, + expectedPDBMinAvailable: 0, + }, + { + name: "create PDB for multi node etcd cluster when none exists", + etcdReplicas: 3, + expectedPDBMinAvailable: 2, + }, + { + name: "returns error when client create fails", + createErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncPodDisruptionBudget, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + syncErr := operator.Sync(opCtx, etcd) + latestPDB, getErr := getLatestPodDisruptionBudget(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } else { + g.Expect(syncErr).ToNot(HaveOccurred()) + g.Expect(getErr).ToNot(HaveOccurred()) + checkPodDisruptionBudget(g, etcd, latestPDB, tc.expectedPDBMinAvailable) + } + }) + } +} + +func TestSyncWhenPDBExists(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + testCases := []struct { + name string + originalEtcdReplicas int32 + updatedEtcdReplicas int32 + expectedPDBMinAvailable int32 + patchErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "successfully update PDB when etcd cluster replicas changed from 1 -> 3", + originalEtcdReplicas: 1, + updatedEtcdReplicas: 3, + expectedPDBMinAvailable: 2, + }, + { + name: "returns error when client patch fails", + originalEtcdReplicas: 1, + updatedEtcdReplicas: 3, + expectedPDBMinAvailable: 0, + patchErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncPodDisruptionBudget, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + existingEtcd := etcdBuilder.WithReplicas(tc.originalEtcdReplicas).Build() + cl := testutils.NewFakeClientBuilder(). + WithPatchError(tc.patchErr). + WithObjects(testsample.NewPodDisruptionBudget(existingEtcd)). + Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + updatedEtcd := etcdBuilder.WithReplicas(tc.updatedEtcdReplicas).Build() + syncErr := operator.Sync(opCtx, updatedEtcd) + latestPDB, getErr := getLatestPodDisruptionBudget(cl, updatedEtcd) + g.Expect(getErr).To(BeNil()) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + } else { + g.Expect(syncErr).To(BeNil()) + } + g.Expect(latestPDB.Spec.MinAvailable.IntVal).To(Equal(tc.expectedPDBMinAvailable)) + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testCases := []struct { + name string + pdbExists bool + deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "no-op and no error if the pdb is not found", + pdbExists: false, + }, + { + name: "successfully deletes existing pdb", + pdbExists: true, + }, + { + name: "returns error when client delete fails", + pdbExists: true, + deleteErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeletePodDisruptionBudget, + Cause: apiInternalErr, + Operation: "TriggerDelete", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) + if tc.pdbExists { + fakeClientBuilder.WithObjects(testsample.NewPodDisruptionBudget(etcd)) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + syncErr := operator.TriggerDelete(opCtx, etcd) + _, getErr := getLatestPodDisruptionBudget(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(getErr).To(BeNil()) + } else { + g.Expect(syncErr).NotTo(HaveOccurred()) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- + +func checkPodDisruptionBudget(g *WithT, etcd *druidv1alpha1.Etcd, actualPDB *policyv1.PodDisruptionBudget, expectedPDBMinAvailable int32) { + g.Expect(actualPDB.Name).To(Equal(etcd.Name)) + g.Expect(actualPDB.Namespace).To(Equal(etcd.Namespace)) + g.Expect(actualPDB.Labels).To(Equal(etcd.GetDefaultLabels())) + expectedAnnotations := map[string]string{ + common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), + common.GardenerOwnerType: "etcd", + } + g.Expect(actualPDB.Annotations).To(Equal(expectedAnnotations)) + g.Expect(actualPDB.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) + g.Expect(*actualPDB.Spec.Selector).To(Equal(metav1.LabelSelector{MatchLabels: etcd.GetDefaultLabels()})) + g.Expect(actualPDB.Spec.MinAvailable.IntVal).To(Equal(expectedPDBMinAvailable)) +} + +func getLatestPodDisruptionBudget(cl client.Client, etcd *druidv1alpha1.Etcd) (*policyv1.PodDisruptionBudget, error) { + pdb := &policyv1.PodDisruptionBudget{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace}, pdb) + return pdb, err +} diff --git a/test/sample/pdb.go b/test/sample/pdb.go new file mode 100644 index 000000000..35cb54a68 --- /dev/null +++ b/test/sample/pdb.go @@ -0,0 +1,43 @@ +package sample + +import ( + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// NewPodDisruptionBudget creates a new instance of PodDisruptionBudget from the passed etcd object. +func NewPodDisruptionBudget(etcd *druidv1alpha1.Etcd) *policyv1.PodDisruptionBudget { + return &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.Name, + Namespace: etcd.Namespace, + Labels: etcd.GetDefaultLabels(), + Annotations: map[string]string{ + common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), + common.GardenerOwnerType: "etcd", + }, + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: etcd.GetDefaultLabels(), + }, + MinAvailable: &intstr.IntOrString{ + IntVal: computePDBMinAvailable(etcd.Spec.Replicas), + Type: intstr.Int, + }, + }, + } +} + +func computePDBMinAvailable(etcdReplicas int32) int32 { + if etcdReplicas <= 1 { + return 0 + } + return etcdReplicas/2 + 1 +} From 60da4a9dc84f53f8e097efe4af5d0975b67d0903 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 11 Jan 2024 16:42:45 +0530 Subject: [PATCH 069/235] added test for snapshot lease and fixed an issue in member lease --- internal/operator/memberlease/memberlease.go | 17 +- .../operator/memberlease/memberlease_test.go | 47 ++- .../operator/peerservice/peerservice_test.go | 21 +- .../operator/snapshotlease/snapshotlease.go | 103 ++++-- .../snapshotlease/snapshotlease_test.go | 305 ++++++++++++++++++ test/sample/etcd.go | 68 ++-- test/sample/pdb.go | 5 +- test/sample/service.go | 18 +- test/sample/snapshotlease.go | 35 ++ 9 files changed, 517 insertions(+), 102 deletions(-) create mode 100644 internal/operator/snapshotlease/snapshotlease_test.go create mode 100644 test/sample/snapshotlease.go diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 738907601..7768b38dd 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -89,18 +89,17 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of member leases") - err := r.client.DeleteAllOf(ctx, + if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(etcd.Namespace), - client.MatchingLabels(etcd.GetDefaultLabels())) - if err == nil { - ctx.Logger.Info("deleted", "resource", "member-leases") - return nil + client.MatchingLabels(getLabels(etcd))); err != nil { + return druiderr.WrapError(err, + ErrDeleteMemberLease, + "TriggerDelete", + fmt.Sprintf("Failed to delete member leases for etcd: %v", etcd.GetNamespaceName())) } - return druiderr.WrapError(err, - ErrDeleteMemberLease, - "TriggerDelete", - fmt.Sprintf("Failed to delete member leases for etcd: %v", etcd.GetNamespaceName())) + ctx.Logger.Info("deleted", "resource", "member-leases") + return nil } func New(client client.Client) resource.Operator { diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 80dbd7654..897a95354 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -196,11 +196,12 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { testCases := []struct { - name string - etcdReplicas int32 // original replicas - numExistingLeases int - deleteAllOfErr *apierrors.StatusError - expectedErr *druiderr.DruidError + name string + etcdReplicas int32 // original replicas + numExistingLeases int + testWithSnapshotLeases bool // this is to ensure that delete of member leases should not delete snapshot leases + deleteAllOfErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { name: "no-op when no member lease exists", @@ -212,6 +213,12 @@ func TestTriggerDelete(t *testing.T) { etcdReplicas: 3, numExistingLeases: 3, }, + { + name: "only delete member leases and not snapshot leases", + etcdReplicas: 3, + numExistingLeases: 3, + testWithSnapshotLeases: true, + }, { name: "successfully deletes remainder member leases", etcdReplicas: 3, @@ -244,6 +251,9 @@ func TestTriggerDelete(t *testing.T) { fakeClientBuilder.WithObjects(lease) } } + if tc.testWithSnapshotLeases { + fakeClientBuilder.WithObjects(testsample.NewDeltaSnapshotLease(etcd), testsample.NewFullSnapshotLease(etcd)) + } cl := fakeClientBuilder.Build() // ***************** Setup operator and test ***************** operator := New(cl) @@ -257,13 +267,16 @@ func TestTriggerDelete(t *testing.T) { g.Expect(memberLeasesPostDelete).Should(HaveLen(tc.numExistingLeases)) } else { g.Expect(memberLeasesPostDelete).Should(HaveLen(0)) + if tc.testWithSnapshotLeases { + snapshotLeases := getLatestSnapshotLeases(g, cl, etcd) + g.Expect(snapshotLeases).To(HaveLen(2)) + } } }) } } // ---------------------------- Helper Functions ----------------------------- - func getAdditionalMemberLeaseLabels(etcdName string) map[string]string { return map[string]string{ common.GardenerOwnedBy: etcdName, @@ -272,11 +285,25 @@ func getAdditionalMemberLeaseLabels(etcdName string) map[string]string { } func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { - leases := &coordinationv1.LeaseList{} - g.Expect(cl.List(context.Background(), leases, client.InNamespace(etcd.Namespace), client.MatchingLabels(map[string]string{ + return doGetLatestLeases(g, cl, etcd, map[string]string{ common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-member-lease", - }))).To(Succeed()) + v1beta1constants.GardenerPurpose: purpose, + }) +} + +func getLatestSnapshotLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { + return doGetLatestLeases(g, cl, etcd, map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", + }) +} + +func doGetLatestLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) []coordinationv1.Lease { + leases := &coordinationv1.LeaseList{} + g.Expect(cl.List(context.Background(), + leases, + client.InNamespace(etcd.Namespace), + client.MatchingLabels(matchingLabels))).To(Succeed()) return leases.Items } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index e2c3a3303..a5d301044 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -236,15 +236,14 @@ func TestPeerServiceTriggerDelete(t *testing.T) { cl := fakeClientBuilder.Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.TriggerDelete(opCtx, etcd) + syncErr := operator.TriggerDelete(opCtx, etcd) + _, getErr := getLatestPeerService(cl, etcd) if tc.expectError != nil { - testutils.CheckDruidError(g, tc.expectError, err) + testutils.CheckDruidError(g, tc.expectError, syncErr) + g.Expect(getErr).ToNot(HaveOccurred()) } else { - g.Expect(err).NotTo(HaveOccurred()) - svc := corev1.Service{} - err = cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace}, &svc) - g.Expect(err).ToNot(BeNil()) - g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + g.Expect(syncErr).NotTo(HaveOccurred()) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } }) } @@ -252,7 +251,8 @@ func TestPeerServiceTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func checkPeerService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { - svc := getLatestPeerService(g, cl, etcd) + svc, err := getLatestPeerService(cl, etcd) + g.Expect(err).ToNot(HaveOccurred()) peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) g.Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) g.Expect(svc.Labels).To(Equal(etcd.GetDefaultLabels())) @@ -271,9 +271,8 @@ func checkPeerService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { )) } -func getLatestPeerService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) *corev1.Service { +func getLatestPeerService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { svc := &corev1.Service{} err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace}, svc) - g.Expect(err).To(BeNil()) - return svc + return svc, err } diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 308fae840..9ccf68c7e 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -2,9 +2,11 @@ package snapshotlease import ( "context" + "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -18,6 +20,12 @@ import ( const purpose = "etcd-snapshot-lease" +const ( + ErrGetSnapshotLease druidv1alpha1.ErrorCode = "ERR_GET_SNAPSHOT_LEASE" + ErrDeleteSnapshotLease druidv1alpha1.ErrorCode = "ERR_DELETE_SNAPSHOT_LEASE" + ErrSyncSnapshotLease druidv1alpha1.ErrorCode = "ERR_SYNC_SNAPSHOT_LEASE" +) + type _resource struct { client client.Client } @@ -31,33 +39,41 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * deltaSnapshotLease, err := r.getLease(ctx, client.ObjectKey{Name: etcd.GetDeltaSnapshotLeaseName(), Namespace: etcd.Namespace}) if err != nil { - return resourceNames, err + return resourceNames, &druiderr.DruidError{ + Code: ErrGetSnapshotLease, + Cause: err, + Operation: "GetExistingResourceNames", + Message: fmt.Sprintf("Error getting delta snapshot lease: %s for etcd: %v", etcd.GetDeltaSnapshotLeaseName(), etcd.GetNamespaceName()), + } + } + if deltaSnapshotLease != nil { + resourceNames = append(resourceNames, deltaSnapshotLease.Name) } - resourceNames = append(resourceNames, deltaSnapshotLease.Name) fullSnapshotLease, err := r.getLease(ctx, client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace}) if err != nil { - return resourceNames, err - } - resourceNames = append(resourceNames, fullSnapshotLease.Name) - return resourceNames, nil -} - -func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*coordinationv1.Lease, error) { - lease := &coordinationv1.Lease{} - if err := r.client.Get(ctx, objectKey, lease); err != nil { - if errors.IsNotFound(err) { - return nil, nil + return resourceNames, &druiderr.DruidError{ + Code: ErrGetSnapshotLease, + Cause: err, + Operation: "GetExistingResourceNames", + Message: fmt.Sprintf("Error getting full snapshot lease: %s for etcd: %v", etcd.GetFullSnapshotLeaseName(), etcd.GetNamespaceName()), } - return nil, err } - return lease, nil + if fullSnapshotLease != nil { + resourceNames = append(resourceNames, fullSnapshotLease.Name) + } + return resourceNames, nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { ctx.Logger.Info("Backup has been disabled. Triggering delete of snapshot leases") - return r.TriggerDelete(ctx, etcd) + return r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { + return druiderr.WrapError(err, + ErrSyncSnapshotLease, + "Sync", + fmt.Sprintf("Failed to delete existing snapshot leases due to backup being disabled for etcd: %v", etcd.GetNamespaceName())) + }) } for _, objKey := range getObjectKeys(etcd) { opResult, err := r.doSync(ctx, etcd, objKey) @@ -69,22 +85,26 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } -func (r _resource) doSync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) (controllerutil.OperationResult, error) { - lease := emptySnapshotLease(leaseObjectKey) - opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { - lease.Labels = getLabels(etcd) - lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - return nil - }) - return opResult, err +func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + ctx.Logger.Info("Triggering delete of snapshot leases") + if err := r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { + return druiderr.WrapError(err, + ErrDeleteSnapshotLease, + "TriggerDelete", + fmt.Sprintf("Failed to delete snapshot leases for etcd: %v", etcd.GetNamespaceName())) + }); err != nil { + return err + } + ctx.Logger.Info("deleted", "resource", "snapshot-leases") + return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - for _, objKey := range getObjectKeys(etcd) { - err := client.IgnoreNotFound(r.client.Delete(ctx, emptySnapshotLease(objKey))) - if err != nil { - return err - } +func (r _resource) deleteAllSnapshotLeases(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, wrapErrFn func(error) error) error { + if err := r.client.DeleteAllOf(ctx, + &coordinationv1.Lease{}, + client.InNamespace(etcd.Namespace), + client.MatchingLabels(getLabels(etcd))); err != nil { + return wrapErrFn(err) } return nil } @@ -95,11 +115,32 @@ func New(client client.Client) resource.Operator { } } +func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*coordinationv1.Lease, error) { + lease := &coordinationv1.Lease{} + if err := r.client.Get(ctx, objectKey, lease); err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return lease, nil +} + +func (r _resource) doSync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) (controllerutil.OperationResult, error) { + lease := emptySnapshotLease(leaseObjectKey) + opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { + lease.Labels = getLabels(etcd) + lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + return nil + }) + return opResult, err +} + func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { labels := make(map[string]string) labels[common.GardenerOwnedBy] = etcd.Name labels[v1beta1constants.GardenerPurpose] = purpose - return utils.MergeMaps[string, string](labels, etcd.GetDefaultLabels()) + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), labels) } func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go new file mode 100644 index 000000000..89c876c66 --- /dev/null +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -0,0 +1,305 @@ +package snapshotlease + +import ( + "context" + "errors" + "fmt" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/go-logr/logr" + "github.com/google/uuid" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + testEtcdName = "test-etcd" + testNs = "test-namespace" +) + +var ( + internalErr = errors.New("test internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs) + testCases := []struct { + name string + backupEnabled bool + getErr *apierrors.StatusError + expectedLeaseNames []string + expectedErr *druiderr.DruidError + }{ + { + name: "no snapshot leases created when backup is disabled", + backupEnabled: false, + expectedLeaseNames: []string{}, + }, + { + name: "successfully returns delta and full snapshot leases", + backupEnabled: true, + expectedLeaseNames: []string{ + fmt.Sprintf("%s-delta-snap", testEtcdName), + fmt.Sprintf("%s-full-snap", testEtcdName), + }, + }, + { + name: "returns error when client get fails", + backupEnabled: true, + getErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetSnapshotLease, + Cause: apiInternalErr, + Operation: "GetExistingResourceNames", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.backupEnabled { + etcdBuilder.WithDefaultBackup() + } + etcd := etcdBuilder.Build() + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + if tc.backupEnabled { + fakeClientBuilder.WithObjects(testsample.NewDeltaSnapshotLease(etcd), testsample.NewFullSnapshotLease(etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + actualSnapshotLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + } else { + g.Expect(err).To(BeNil()) + g.Expect(actualSnapshotLeaseNames, tc.expectedLeaseNames) + } + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSyncWhenBackupIsEnabled(t *testing.T) { + etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + testCases := []struct { + name string + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "create snapshot lease when backup is enabled", + }, + { + name: "returns error when client create fails", + createErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncSnapshotLease, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + syncErr := operator.Sync(opCtx, etcd) + latestSnapshotLeases, listErr := getLatestSnapshotLeases(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(listErr).ToNot(HaveOccurred()) + g.Expect(latestSnapshotLeases).To(BeEmpty()) + } else { + g.Expect(listErr).ToNot(HaveOccurred()) + g.Expect(latestSnapshotLeases).To(ConsistOf(matchLease(etcd.GetDeltaSnapshotLeaseName(), etcd), matchLease(etcd.GetFullSnapshotLeaseName(), etcd))) + } + }) + } +} + +func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { + existingEtcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() // backup is enabled + updatedEtcd := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs).Build() // backup is disabled + testCases := []struct { + name string + deleteAllOfErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "deletes snapshot leases when backup has been disabled", + }, + { + name: "returns error when client delete fails", + deleteAllOfErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncSnapshotLease, + Cause: apiInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := testutils.NewFakeClientBuilder(). + WithDeleteAllOfError(tc.deleteAllOfErr). + WithObjects(testsample.NewDeltaSnapshotLease(existingEtcd), testsample.NewFullSnapshotLease(existingEtcd)). + Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + syncErr := operator.Sync(opCtx, updatedEtcd) + latestSnapshotLeases, listErr := getLatestSnapshotLeases(cl, updatedEtcd) + g.Expect(listErr).ToNot(HaveOccurred()) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(latestSnapshotLeases).To(HaveLen(2)) + } else { + g.Expect(latestSnapshotLeases).To(HaveLen(0)) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + etcdBuilder := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs).WithReplicas(3) + testCases := []struct { + name string + backupEnabled bool + deleteAllErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "no-op when backup is not enabled", + backupEnabled: false, + }, + { + name: "should only delete snapshot leases when backup is enabled", + backupEnabled: true, + }, + { + name: "should return error when client delete-all fails", + backupEnabled: true, + deleteAllErr: apiInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeleteSnapshotLease, + Cause: apiInternalErr, + Operation: "TriggerDelete", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.backupEnabled { + etcdBuilder.WithDefaultBackup() + } + etcd := etcdBuilder.Build() + fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllErr) + memberLeases, err := testsample.NewMemberLeases(etcd, int(etcd.Spec.Replicas), map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-member-lease", + }) + g.Expect(err).ToNot(HaveOccurred()) + for _, lease := range memberLeases { + fakeClientBuilder.WithObjects(lease) + } + if tc.backupEnabled { + fakeClientBuilder.WithObjects(testsample.NewDeltaSnapshotLease(etcd), testsample.NewFullSnapshotLease(etcd)) + } + cl := fakeClientBuilder.Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + latestSnapshotLeases, snapshotLeaseListErr := getLatestSnapshotLeases(cl, etcd) + latestMemberLeases, memberLeaseListErr := getLatestMemberLeases(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) + g.Expect(snapshotLeaseListErr).ToNot(HaveOccurred()) + g.Expect(latestSnapshotLeases).To(HaveLen(2)) + g.Expect(memberLeaseListErr).ToNot(HaveOccurred()) + g.Expect(latestMemberLeases).To(HaveLen(int(etcd.Spec.Replicas))) + } else { + g.Expect(triggerDeleteErr).ToNot(HaveOccurred()) + g.Expect(snapshotLeaseListErr).ToNot(HaveOccurred()) + g.Expect(latestSnapshotLeases).To(HaveLen(0)) + g.Expect(memberLeaseListErr).ToNot(HaveOccurred()) + g.Expect(latestMemberLeases).To(HaveLen(int(etcd.Spec.Replicas))) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- + +func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { + expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: purpose, + }) + return MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(leaseName), + "Namespace": Equal(etcd.Namespace), + "Labels": Equal(expectedLabels), + "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ + "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), + "Kind": Equal("Etcd"), + "Name": Equal(etcd.Name), + "UID": Equal(etcd.UID), + "Controller": PointTo(BeTrue()), + "BlockOwnerDeletion": PointTo(BeTrue()), + })), + }), + }) +} + +func getLatestMemberLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { + return doGetLatestLeases(cl, etcd, map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-member-lease", + }) +} + +func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { + return doGetLatestLeases(cl, etcd, map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: purpose, + }) +} + +func doGetLatestLeases(cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) ([]coordinationv1.Lease, error) { + leases := &coordinationv1.LeaseList{} + err := cl.List(context.Background(), + leases, + client.InNamespace(etcd.Namespace), + client.MatchingLabels(matchingLabels)) + if err != nil { + return nil, err + } + return leases.Items, nil +} diff --git a/test/sample/etcd.go b/test/sample/etcd.go index 4ec452c11..7157cce91 100644 --- a/test/sample/etcd.go +++ b/test/sample/etcd.go @@ -293,6 +293,12 @@ func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { return eb } +// WithDefaultBackup creates a default backup spec and initializes etcd with it. +func (eb *EtcdBuilder) WithDefaultBackup() *EtcdBuilder { + eb.etcd.Spec.Backup = getBackupSpec() + return eb +} + func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { return eb.etcd } @@ -361,35 +367,7 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { StorageClass: &storageClass, PriorityClassName: &priorityClassName, VolumeClaimTemplate: &volumeClaimTemplateName, - Backup: druidv1alpha1.BackupSpec{ - Image: &imageBR, - Port: &backupPort, - FullSnapshotSchedule: &snapshotSchedule, - GarbageCollectionPolicy: &garbageCollectionPolicy, - GarbageCollectionPeriod: &garbageCollectionPeriod, - DeltaSnapshotPeriod: &deltaSnapshotPeriod, - DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, - EtcdSnapshotTimeout: &etcdSnapshotTimeout, - - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("2Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("23m"), - "memory": testutils.ParseQuantity("128Mi"), - }, - }, - Store: &druidv1alpha1.StoreSpec{ - SecretRef: &corev1.SecretReference{ - Name: "etcd-backup", - }, - Container: &container, - Provider: &localProvider, - Prefix: prefix, - }, - }, + Backup: getBackupSpec(), Etcd: druidv1alpha1.EtcdConfig{ Quota: "a, Metrics: &metricsBasic, @@ -417,6 +395,38 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { } } +func getBackupSpec() druidv1alpha1.BackupSpec { + return druidv1alpha1.BackupSpec{ + Image: &imageBR, + Port: &backupPort, + FullSnapshotSchedule: &snapshotSchedule, + GarbageCollectionPolicy: &garbageCollectionPolicy, + GarbageCollectionPeriod: &garbageCollectionPeriod, + DeltaSnapshotPeriod: &deltaSnapshotPeriod, + DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, + EtcdSnapshotTimeout: &etcdSnapshotTimeout, + + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": testutils.ParseQuantity("500m"), + "memory": testutils.ParseQuantity("2Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": testutils.ParseQuantity("23m"), + "memory": testutils.ParseQuantity("128Mi"), + }, + }, + Store: &druidv1alpha1.StoreSpec{ + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + Container: &container, + Provider: &localProvider, + Prefix: prefix, + }, + } +} + func getBackupStore(name string, provider druidv1alpha1.StorageProvider) *druidv1alpha1.StoreSpec { return &druidv1alpha1.StoreSpec{ Container: &container, diff --git a/test/sample/pdb.go b/test/sample/pdb.go index 35cb54a68..99544fb18 100644 --- a/test/sample/pdb.go +++ b/test/sample/pdb.go @@ -4,7 +4,6 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -18,8 +17,8 @@ func NewPodDisruptionBudget(etcd *druidv1alpha1.Etcd) *policyv1.PodDisruptionBud Namespace: etcd.Namespace, Labels: etcd.GetDefaultLabels(), Annotations: map[string]string{ - common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - common.GardenerOwnerType: "etcd", + "gardener.cloud/owned-by": fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), + "gardener.cloud/owner-type": "etcd", }, OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, }, diff --git a/test/sample/service.go b/test/sample/service.go index b6972e53d..8cc1b0da0 100644 --- a/test/sample/service.go +++ b/test/sample/service.go @@ -61,27 +61,27 @@ func NewPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { } func getClientServicePorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - backupPort := testutils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) - clientPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) - peerPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + svcBackupPort := testutils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) + svcClientPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) + svcPeerPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) return []corev1.ServicePort{ { Name: "client", Protocol: corev1.ProtocolTCP, - Port: clientPort, - TargetPort: intstr.FromInt(int(clientPort)), + Port: svcClientPort, + TargetPort: intstr.FromInt(int(svcClientPort)), }, { Name: "server", Protocol: corev1.ProtocolTCP, - Port: peerPort, - TargetPort: intstr.FromInt(int(peerPort)), + Port: svcPeerPort, + TargetPort: intstr.FromInt(int(svcPeerPort)), }, { Name: "backuprestore", Protocol: corev1.ProtocolTCP, - Port: backupPort, - TargetPort: intstr.FromInt(int(backupPort)), + Port: svcBackupPort, + TargetPort: intstr.FromInt(int(svcBackupPort)), }, } } diff --git a/test/sample/snapshotlease.go b/test/sample/snapshotlease.go new file mode 100644 index 000000000..0b7eb68e9 --- /dev/null +++ b/test/sample/snapshotlease.go @@ -0,0 +1,35 @@ +package sample + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NewDeltaSnapshotLease creates a delta snapshot lease from the passed in etcd object. +func NewDeltaSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { + leaseName := etcd.GetDeltaSnapshotLeaseName() + return createLease(etcd, leaseName) +} + +// NewFullSnapshotLease creates a full snapshot lease from the passed in etcd object. +func NewFullSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { + leaseName := etcd.GetFullSnapshotLeaseName() + return createLease(etcd, leaseName) +} + +func createLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: leaseName, + Namespace: etcd.Namespace, + Labels: utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + "gardener.cloud/owned-by": etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", + }), + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + } +} From 39eb8786a970ef8ef56c9d8d03bb28586f4173a1 Mon Sep 17 00:00:00 2001 From: seshachalam-yv Date: Fri, 12 Jan 2024 18:28:58 +0530 Subject: [PATCH 070/235] Fix assertion in all operator tests to correctly compare names --- .../clientservice/clientservice_test.go | 2 +- .../operator/memberlease/memberlease_test.go | 5 ++- .../operator/peerservice/peerservice_test.go | 2 +- .../poddisruptionbudget_test.go | 2 +- internal/operator/role/role_test.go | 2 +- .../operator/rolebinding/rolebinding_test.go | 2 +- .../operator/snapshotlease/snapshotlease.go | 39 ++++++++++++++----- .../snapshotlease/snapshotlease_test.go | 2 +- 8 files changed, 39 insertions(+), 17 deletions(-) diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index eb229f81b..5761f9324 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -98,8 +98,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) + g.Expect(svcNames).To(Equal(tc.expectedServiceNames)) } - g.Expect(svcNames, tc.expectedServiceNames) }) } } diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 897a95354..6e0d9ed79 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -88,7 +88,7 @@ func TestGetExistingResourceNames(t *testing.T) { WithGetError(tc.getErr). WithListError(tc.listErr) if tc.numExistingLeases > 0 { - leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Namespace)) + leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { fakeClientBuilder.WithObjects(lease) @@ -101,7 +101,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) - g.Expect(len(memberLeaseNames), tc.numExistingLeases) + expectedLeaseNames := etcd.GetMemberLeaseNames()[:tc.numExistingLeases] + g.Expect(memberLeaseNames).To(Equal(expectedLeaseNames)) } }) } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index a5d301044..b84c1f0e4 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -96,8 +96,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) + g.Expect(svcNames).To(Equal(tc.expectedServiceNames)) } - g.Expect(svcNames, tc.expectedServiceNames) }) } } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 19c419b95..0d3114d61 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -82,8 +82,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) + g.Expect(pdbNames).To(Equal(tc.expectedPDBNames)) } - g.Expect(pdbNames, tc.expectedPDBNames) }) } } diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 70ece7219..2dd9f6e1e 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -82,8 +82,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) + g.Expect(roleNames).To(Equal(tc.expectedRoleNames)) } - g.Expect(roleNames, tc.expectedRoleNames) }) } } diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index f8978a8f7..bbaf392ce 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -83,8 +83,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) + g.Expect(roleBindingNames).To(Equal(tc.expectedRoleBindingNames)) } - g.Expect(roleBindingNames, tc.expectedRoleBindingNames) }) } } diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 9ccf68c7e..6c839a663 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -11,11 +11,11 @@ import ( "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" + "github.com/hashicorp/go-multierror" coordinationv1 "k8s.io/api/coordination/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const purpose = "etcd-snapshot-lease" @@ -75,14 +75,27 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Failed to delete existing snapshot leases due to backup being disabled for etcd: %v", etcd.GetNamespaceName())) }) } - for _, objKey := range getObjectKeys(etcd) { - opResult, err := r.doSync(ctx, etcd, objKey) - if err != nil { - return err + + objectKeys := getObjectKeys(etcd) + createTasks := make([]utils.OperatorTask, len(objectKeys)) + var errs error + + for i, objKey := range objectKeys { + objKey := objKey // capture the range variable + createTasks[i] = utils.OperatorTask{ + Name: "CreateOrUpdate-" + objKey.String(), + Fn: func(ctx resource.OperatorContext) error { + return r.doCreateOrUpdate(ctx, etcd, objKey) + }, } - ctx.Logger.Info("Triggered create or update", "lease", objKey, "result", opResult) } - return nil + + if errorList := utils.RunConcurrently(ctx, createTasks); len(errorList) > 0 { + for _, err := range errorList { + errs = multierror.Append(errs, err) + } + } + return errs } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { @@ -126,14 +139,22 @@ func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*c return lease, nil } -func (r _resource) doSync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) (controllerutil.OperationResult, error) { +func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) error { lease := emptySnapshotLease(leaseObjectKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { lease.Labels = getLabels(etcd) lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} return nil }) - return opResult, err + if err != nil { + return druiderr.WrapError(err, + ErrSyncSnapshotLease, + "Sync", + fmt.Sprintf("Error syncing snapshot lease: %s for etcd: %v", leaseObjectKey.Name, etcd.GetNamespaceName())) + } + ctx.Logger.Info("triggered create or update of snapshot lease", "lease", leaseObjectKey, "operationResult", opResult) + + return nil } func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 89c876c66..3a9730ef2 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -88,7 +88,7 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) - g.Expect(actualSnapshotLeaseNames, tc.expectedLeaseNames) + g.Expect(actualSnapshotLeaseNames).To(Equal(tc.expectedLeaseNames)) } }) } From bb2f833622e6f33b41ce16d80f530377820a8416 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sun, 14 Jan 2024 15:16:30 +0530 Subject: [PATCH 071/235] introduced utility for custom gomega matcher and adapted client and peer service tests --- .../operator/clientservice/clientservice.go | 19 +-- .../clientservice/clientservice_test.go | 121 +++++++++++------- internal/operator/peerservice/peerservice.go | 20 +-- .../operator/peerservice/peerservice_test.go | 78 ++++++----- test/sample/service.go | 87 ------------- test/utils/matcher.go | 87 +++++++++++++ 6 files changed, 236 insertions(+), 176 deletions(-) delete mode 100644 test/sample/service.go create mode 100644 test/utils/matcher.go diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index c39663397..96ec436e6 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -52,14 +52,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { svc := emptyClientService(getObjectKey(etcd)) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { - svc.Labels = getLabels(etcd) - svc.Annotations = getAnnotations(etcd) - svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - svc.Spec.Type = corev1.ServiceTypeClusterIP - svc.Spec.SessionAffinity = corev1.ServiceAffinityNone - svc.Spec.Selector = etcd.GetDefaultLabels() - svc.Spec.Ports = getPorts(etcd) - + buildResource(etcd, svc) return nil }) if err == nil { @@ -94,6 +87,16 @@ func New(client client.Client) resource.Operator { } } +func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { + svc.Labels = getLabels(etcd) + svc.Annotations = getAnnotations(etcd) + svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Spec.SessionAffinity = corev1.ServiceAffinityNone + svc.Spec.Selector = etcd.GetDefaultLabels() + svc.Spec.Ports = getPorts(etcd) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{ Name: etcd.GetClientServiceName(), diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 5761f9324..5fa9adfb0 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -21,11 +21,11 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" + . "github.com/onsi/gomega/gstruct" "k8s.io/utils/pointer" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - "github.com/gardener/etcd-druid/test/sample" testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -34,7 +34,6 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -89,7 +88,7 @@ func TestGetExistingResourceNames(t *testing.T) { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewClientService(etcd)) + fakeClientBuilder.WithObjects(newClientService(etcd)) } operator := New(fakeClientBuilder.Build()) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -142,17 +141,25 @@ func TestSyncWhenNoServiceExists(t *testing.T) { operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) + latestClientSvc, getErr := getLatestClientService(cl, etcd) if tc.expectedError != nil { testutils.CheckDruidError(g, tc.expectedError, err) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } else { g.Expect(err).NotTo(HaveOccurred()) - checkClientService(g, cl, etcd) + matchClientService(g, etcd, *latestClientSvc) } }) } } func TestSyncWhenServiceExists(t *testing.T) { + const ( + originalClientPort = 2379 + originalServerPort = 2380 + originalBackupPort = 8080 + ) + existingEtcd := buildEtcd(pointer.Int32(originalClientPort), pointer.Int32(originalServerPort), pointer.Int32(originalBackupPort)) testCases := []struct { name string clientPort *int32 @@ -181,19 +188,25 @@ func TestSyncWhenServiceExists(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - existingEtcd := buildEtcd(nil, nil, nil) - cl := testutils.NewFakeClientBuilder().WithPatchError(tc.patchErr). - WithObjects(sample.NewClientService(existingEtcd)). + // ********************* Setup ********************* + cl := testutils.NewFakeClientBuilder(). + WithPatchError(tc.patchErr). + WithObjects(newClientService(existingEtcd)). Build() + // ********************* test sync with updated ports ********************* operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) - err := operator.Sync(opCtx, updatedEtcd) + syncErr := operator.Sync(opCtx, updatedEtcd) + latestClientSvc, getErr := getLatestClientService(cl, updatedEtcd) + g.Expect(latestClientSvc).ToNot(BeNil()) if tc.expectedError != nil { - testutils.CheckDruidError(g, tc.expectedError, err) + testutils.CheckDruidError(g, tc.expectedError, syncErr) + g.Expect(getErr).ToNot(HaveOccurred()) + matchClientService(g, existingEtcd, *latestClientSvc) } else { - g.Expect(err).NotTo(HaveOccurred()) - checkClientService(g, cl, updatedEtcd) + g.Expect(syncErr).NotTo(HaveOccurred()) + matchClientService(g, updatedEtcd, *latestClientSvc) } }) } @@ -231,14 +244,17 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + // ********************* Setup ********************* etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) - if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewClientService(testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build())) - } - cl := fakeClientBuilder.Build() + cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + if tc.svcExists { + syncErr := operator.Sync(opCtx, etcd) + g.Expect(syncErr).ToNot(HaveOccurred()) + ensureClientServiceExists(g, cl, etcd) + } + // ********************* Test trigger delete ********************* triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestClientService, getErr := getLatestClientService(cl, etcd) if tc.expectError != nil { @@ -268,9 +284,7 @@ func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { return etcdBuilder.Build() } -func checkClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { - svc, err := getLatestClientService(cl, etcd) - g.Expect(err).To(BeNil()) +func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) @@ -281,32 +295,53 @@ func checkClientService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) } - g.Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(svc.Annotations).To(Equal(expectedAnnotations)) - g.Expect(svc.Labels).To(Equal(expectedLabels)) - g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) - g.Expect(svc.Spec.SessionAffinity).To(Equal(corev1.ServiceAffinityNone)) - g.Expect(svc.Spec.Selector).To(Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.Ports).To(ConsistOf( - Equal(corev1.ServicePort{ - Name: "client", - Protocol: corev1.ProtocolTCP, - Port: clientPort, - TargetPort: intstr.FromInt(int(clientPort)), - }), - Equal(corev1.ServicePort{ - Name: "server", - Protocol: corev1.ProtocolTCP, - Port: peerPort, - TargetPort: intstr.FromInt(int(peerPort)), + + g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(etcd.GetClientServiceName()), + "Namespace": Equal(etcd.Namespace), + "Annotations": testutils.MatchResourceAnnotations(expectedAnnotations), + "Labels": testutils.MatchResourceLabels(expectedLabels), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), - Equal(corev1.ServicePort{ - Name: "backuprestore", - Protocol: corev1.ProtocolTCP, - Port: backupPort, - TargetPort: intstr.FromInt(int(backupPort)), + "Spec": MatchFields(IgnoreExtras, Fields{ + "Type": Equal(corev1.ServiceTypeClusterIP), + "SessionAffinity": Equal(corev1.ServiceAffinityNone), + "Selector": Equal(etcd.GetDefaultLabels()), + "Ports": ConsistOf( + Equal(corev1.ServicePort{ + Name: "client", + Protocol: corev1.ProtocolTCP, + Port: clientPort, + TargetPort: intstr.FromInt(int(clientPort)), + }), + Equal(corev1.ServicePort{ + Name: "server", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }), + Equal(corev1.ServicePort{ + Name: "backuprestore", + Protocol: corev1.ProtocolTCP, + Port: backupPort, + TargetPort: intstr.FromInt(int(backupPort)), + }), + ), }), - )) + })) +} + +func newClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { + svc := emptyClientService(getObjectKey(etcd)) + buildResource(etcd, svc) + return svc +} + +func ensureClientServiceExists(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { + svc, err := getLatestClientService(cl, etcd) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(svc).ToNot(BeNil()) } func getLatestClientService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 27964f864..278a0b334 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -47,14 +47,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { svc := emptyPeerService(getObjectKey(etcd)) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { - svc.Labels = etcd.GetDefaultLabels() - svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - svc.Spec.Type = corev1.ServiceTypeClusterIP - svc.Spec.ClusterIP = corev1.ClusterIPNone - svc.Spec.SessionAffinity = corev1.ServiceAffinityNone - svc.Spec.Selector = etcd.GetDefaultLabels() - svc.Spec.PublishNotReadyAddresses = true - svc.Spec.Ports = getPorts(etcd) + buildResource(etcd, svc) return nil }) if err == nil { @@ -90,6 +83,17 @@ func New(client client.Client) resource.Operator { } } +func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { + svc.Labels = etcd.GetDefaultLabels() + svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Spec.ClusterIP = corev1.ClusterIPNone + svc.Spec.SessionAffinity = corev1.ServiceAffinityNone + svc.Spec.Selector = etcd.GetDefaultLabels() + svc.Spec.PublishNotReadyAddresses = true + svc.Spec.Ports = getPorts(etcd) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace} } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index b84c1f0e4..aebc564a3 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -23,15 +23,14 @@ import ( druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - "github.com/gardener/etcd-druid/test/sample" testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -87,7 +86,7 @@ func TestGetExistingResourceNames(t *testing.T) { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewPeerService(etcd)) + fakeClientBuilder.WithObjects(newPeerService(etcd)) } operator := New(fakeClientBuilder.Build()) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -139,12 +138,16 @@ func TestSyncWhenNoServiceExists(t *testing.T) { etcd := etcdBuilder.Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.Sync(opCtx, etcd) + syncErr := operator.Sync(opCtx, etcd) + latestPeerService, getErr := getLatestPeerService(cl, etcd) if tc.expectedError != nil { - testutils.CheckDruidError(g, tc.expectedError, err) + testutils.CheckDruidError(g, tc.expectedError, syncErr) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } else { - g.Expect(err).NotTo(HaveOccurred()) - checkPeerService(g, cl, etcd) + g.Expect(syncErr).NotTo(HaveOccurred()) + g.Expect(getErr).ToNot(HaveOccurred()) + g.Expect(latestPeerService).ToNot(BeNil()) + matchPeerService(g, etcd, *latestPeerService) } }) } @@ -179,17 +182,20 @@ func TestSyncWhenServiceExists(t *testing.T) { t.Run(tc.name, func(t *testing.T) { existingEtcd := etcdBuilder.Build() cl := testutils.NewFakeClientBuilder().WithPatchError(tc.patchErr). - WithObjects(sample.NewPeerService(existingEtcd)). + WithObjects(newPeerService(existingEtcd)). Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithEtcdServerPort(tc.updateWithPort).Build() - err := operator.Sync(opCtx, updatedEtcd) + syncErr := operator.Sync(opCtx, updatedEtcd) + latestPeerService, getErr := getLatestPeerService(cl, updatedEtcd) if tc.expectedError != nil { - testutils.CheckDruidError(g, tc.expectedError, err) + testutils.CheckDruidError(g, tc.expectedError, syncErr) + g.Expect(getErr).ToNot(HaveOccurred()) } else { - g.Expect(err).NotTo(HaveOccurred()) - checkPeerService(g, cl, updatedEtcd) + g.Expect(syncErr).NotTo(HaveOccurred()) + g.Expect(latestPeerService).ToNot(BeNil()) + matchPeerService(g, updatedEtcd, *latestPeerService) } }) } @@ -231,7 +237,7 @@ func TestPeerServiceTriggerDelete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.svcExists { - fakeClientBuilder.WithObjects(sample.NewPeerService(etcd)) + fakeClientBuilder.WithObjects(newPeerService(etcd)) } cl := fakeClientBuilder.Build() operator := New(cl) @@ -250,25 +256,37 @@ func TestPeerServiceTriggerDelete(t *testing.T) { } // ---------------------------- Helper Functions ----------------------------- -func checkPeerService(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { - svc, err := getLatestPeerService(cl, etcd) - g.Expect(err).ToNot(HaveOccurred()) + +func newPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { + svc := emptyPeerService(getObjectKey(etcd)) + buildResource(etcd, svc) + return svc +} + +func matchPeerService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) - g.Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(svc.Labels).To(Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.PublishNotReadyAddresses).To(BeTrue()) - g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) - g.Expect(svc.Spec.ClusterIP).To(Equal(corev1.ClusterIPNone)) - g.Expect(svc.Spec.SessionAffinity).To(Equal(corev1.ServiceAffinityNone)) - g.Expect(svc.Spec.Selector).To(Equal(etcd.GetDefaultLabels())) - g.Expect(svc.Spec.Ports).To(ConsistOf( - Equal(corev1.ServicePort{ - Name: "peer", - Protocol: corev1.ProtocolTCP, - Port: peerPort, - TargetPort: intstr.FromInt(int(peerPort)), + g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(etcd.GetPeerServiceName()), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), + }), + "Spec": MatchFields(IgnoreExtras, Fields{ + "Type": Equal(corev1.ServiceTypeClusterIP), + "ClusterIP": Equal(corev1.ClusterIPNone), + "SessionAffinity": Equal(corev1.ServiceAffinityNone), + "Selector": Equal(etcd.GetDefaultLabels()), + "Ports": ConsistOf( + Equal(corev1.ServicePort{ + Name: "peer", + Protocol: corev1.ProtocolTCP, + Port: peerPort, + TargetPort: intstr.FromInt(int(peerPort)), + }), + ), }), - )) + })) } func getLatestPeerService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { diff --git a/test/sample/service.go b/test/sample/service.go deleted file mode 100644 index 8cc1b0da0..000000000 --- a/test/sample/service.go +++ /dev/null @@ -1,87 +0,0 @@ -package sample - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - testutils "github.com/gardener/etcd-druid/test/utils" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const ( - defaultBackupPort = 8080 - defaultClientPort = 2379 - defaultServerPort = 2380 -) - -// NewClientService creates a new sample client service initializing it from the passed in etcd object. -func NewClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetClientServiceName(), - Namespace: etcd.Namespace, - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - Spec: corev1.ServiceSpec{ - Ports: getClientServicePorts(etcd), - Selector: etcd.GetDefaultLabels(), - Type: corev1.ServiceTypeClusterIP, - SessionAffinity: corev1.ServiceAffinityNone, - }, - } -} - -// NewPeerService creates a new sample peer service initializing it from the passed in etcd object. -func NewPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetPeerServiceName(), - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReferences: []metav1.OwnerReference{ - etcd.GetAsOwnerReference(), - }, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, - SessionAffinity: corev1.ServiceAffinityNone, - Selector: etcd.GetDefaultLabels(), - PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "peer", - Protocol: corev1.ProtocolTCP, - Port: *etcd.Spec.Etcd.ServerPort, - TargetPort: intstr.FromInt(int(*etcd.Spec.Etcd.ServerPort)), - }, - }, - }, - } -} - -func getClientServicePorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - svcBackupPort := testutils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) - svcClientPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) - svcPeerPort := testutils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) - return []corev1.ServicePort{ - { - Name: "client", - Protocol: corev1.ProtocolTCP, - Port: svcClientPort, - TargetPort: intstr.FromInt(int(svcClientPort)), - }, - { - Name: "server", - Protocol: corev1.ProtocolTCP, - Port: svcPeerPort, - TargetPort: intstr.FromInt(int(svcPeerPort)), - }, - { - Name: "backuprestore", - Protocol: corev1.ProtocolTCP, - Port: svcBackupPort, - TargetPort: intstr.FromInt(int(svcBackupPort)), - }, - } -} diff --git a/test/utils/matcher.go b/test/utils/matcher.go new file mode 100644 index 000000000..6f14084a1 --- /dev/null +++ b/test/utils/matcher.go @@ -0,0 +1,87 @@ +package utils + +import ( + "fmt" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + "k8s.io/apimachinery/pkg/types" +) + +// mapMatcher matches map[string]string and produces a convenient and easy to consume error message which includes +// the difference between the actual and expected. +type mapMatcher struct { + fieldName string + expected map[string]string + diff []string +} + +func (m mapMatcher) Match(actual interface{}) (bool, error) { + if actual == nil { + return false, nil + } + actualMap, okType := actual.(map[string]string) + if !okType { + return false, fmt.Errorf("expected a map[string]string. got: %s", format.Object(actual, 1)) + } + for k, v := range m.expected { + actualVal, ok := actualMap[k] + if !ok { + m.diff = append(m.diff, fmt.Sprintf("expected key: %s to be present", k)) + } + if v != actualVal { + m.diff = append(m.diff, fmt.Sprintf("expected val: %s for key; %s, found val:%s instead", v, k, actualVal)) + } + } + return len(m.diff) == 0, nil +} + +func (m mapMatcher) FailureMessage(actual interface{}) string { + return m.createMessage(actual, "to be") +} + +func (m mapMatcher) NegatedFailureMessage(actual interface{}) string { + return m.createMessage(actual, "to not be") +} + +func (m mapMatcher) createMessage(actual interface{}, message string) string { + msgBuilder := strings.Builder{} + msgBuilder.WriteString(format.Message(actual, message, m.expected)) + if len(m.diff) > 0 { + msgBuilder.WriteString(fmt.Sprintf("\nFound difference:\n")) + msgBuilder.WriteString(strings.Join(m.diff, "\n")) + } + return msgBuilder.String() +} + +// MatchResourceAnnotations returns a custom gomega matcher which matches annotations set on a resource against the expected annotations. +func MatchResourceAnnotations(expected map[string]string) gomegatypes.GomegaMatcher { + return &mapMatcher{ + fieldName: "ObjectMeta.Annotations", + expected: expected, + } +} + +// MatchResourceLabels returns a custom gomega matcher which matches labels set on the resource against the expected labels. +func MatchResourceLabels(expected map[string]string) gomegatypes.GomegaMatcher { + return &mapMatcher{ + fieldName: "ObjectMeta.Labels", + expected: expected, + } +} + +// MatchEtcdOwnerReference is a custom gomega matcher which creates a matcher for ObjectMeta.OwnerReferences +func MatchEtcdOwnerReference(etcdName string, etcdUID types.UID) gomegatypes.GomegaMatcher { + return ConsistOf(MatchFields(IgnoreExtras, Fields{ + "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), + "Kind": Equal("Etcd"), + "Name": Equal(etcdName), + "UID": Equal(etcdUID), + "Controller": PointTo(BeTrue()), + "BlockOwnerDeletion": PointTo(BeTrue()), + })) +} From 47aa8d44a80df87bb20bc04966a6dfad163d7605 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 15 Jan 2024 14:34:18 +0530 Subject: [PATCH 072/235] removed role and rolebinding sample and adjusted unit tests --- internal/operator/role/role.go | 10 +- internal/operator/role/role_test.go | 129 +++++++++-------- internal/operator/rolebinding/rolebinding.go | 32 +++-- .../operator/rolebinding/rolebinding_test.go | 131 +++++++++--------- test/sample/role.go | 39 ------ test/sample/rolebinding.go | 30 ---- 6 files changed, 149 insertions(+), 222 deletions(-) delete mode 100644 test/sample/role.go delete mode 100644 test/sample/rolebinding.go diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 8d634e768..205c1ad5c 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -43,9 +43,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { role := emptyRole(etcd) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { - role.Labels = etcd.GetDefaultLabels() - role.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - role.Rules = createPolicyRules() + buildResource(etcd, role) return nil }) if err == nil { @@ -90,8 +88,10 @@ func emptyRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { } } -func createPolicyRules() []rbacv1.PolicyRule { - return []rbacv1.PolicyRule{ +func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { + role.Labels = etcd.GetDefaultLabels() + role.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + role.Rules = []rbacv1.PolicyRule{ { APIGroups: []string{"coordination.k8s.io"}, Resources: []string{"leases"}, diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 2dd9f6e1e..03e250ff7 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -13,10 +13,10 @@ import ( "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -73,7 +73,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithGetError(tc.getErr) } if tc.roleExists { - fakeClientBuilder.WithObjects(testsample.NewRole(etcd)) + fakeClientBuilder.WithObjects(newRole(etcd)) } operator := New(fakeClientBuilder.Build()) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -94,29 +94,15 @@ func TestSync(t *testing.T) { internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string - roleExists bool - getErr *apierrors.StatusError createErr *apierrors.StatusError expectedErr *druiderr.DruidError }{ { - name: "create role when none exists", - roleExists: false, - }, - { - name: "create role fails when client create fails", - roleExists: false, - createErr: internalStatusErr, - expectedErr: &druiderr.DruidError{ - Code: ErrSyncRole, - Cause: internalStatusErr, - Operation: "Sync", - }, + name: "create role when none exists", }, { - name: "create role fails when get errors out", - roleExists: false, - getErr: internalStatusErr, + name: "create role fails when client create fails", + createErr: internalStatusErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncRole, Cause: internalStatusErr, @@ -130,29 +116,21 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder(). - WithGetError(tc.getErr). - WithCreateError(tc.createErr) - if tc.roleExists { - fakeClientBuilder.WithObjects(testsample.NewRole(etcd)) - } - cl := fakeClientBuilder.Build() + cl := testutils.NewFakeClientBuilder(). + WithCreateError(tc.createErr). + Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.Sync(opCtx, etcd) + syncErr := operator.Sync(opCtx, etcd) + latestRole, getErr := getLatestRole(cl, etcd) if tc.expectedErr != nil { - testutils.CheckDruidError(g, tc.expectedErr, err) + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } else { - g.Expect(err).ToNot(HaveOccurred()) - existingRole := &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetRoleName(), - Namespace: etcd.Namespace, - }, - } - err = cl.Get(context.Background(), client.ObjectKeyFromObject(existingRole), existingRole) - g.Expect(err).ToNot(HaveOccurred()) - checkRole(g, existingRole, etcd) + g.Expect(syncErr).ToNot(HaveOccurred()) + g.Expect(getErr).ToNot(HaveOccurred()) + g.Expect(latestRole).ToNot(BeNil()) + matchRole(g, etcd, *latestRole) } }) } @@ -198,44 +176,63 @@ func TestTriggerDelete(t *testing.T) { fakeClientBuilder.WithDeleteError(tc.deleteErr) } if tc.roleExists { - fakeClientBuilder.WithObjects(testsample.NewRole(etcd)) + fakeClientBuilder.WithObjects(newRole(etcd)) } cl := fakeClientBuilder.Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.TriggerDelete(opCtx, etcd) + deleteErr := operator.TriggerDelete(opCtx, etcd) + latestRole, getErr := getLatestRole(cl, etcd) if tc.expectedErr != nil { - testutils.CheckDruidError(g, tc.expectedErr, err) + testutils.CheckDruidError(g, tc.expectedErr, deleteErr) + g.Expect(getErr).ToNot(HaveOccurred()) + g.Expect(latestRole).ToNot(BeNil()) } else { - g.Expect(err).NotTo(HaveOccurred()) - existingRole := rbacv1.Role{} - err = cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace}, &existingRole) - g.Expect(err).To(HaveOccurred()) - g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + g.Expect(deleteErr).NotTo(HaveOccurred()) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } }) } } // ---------------------------- Helper Functions ----------------------------- -func checkRole(g *WithT, role *rbacv1.Role, etcd *druidv1alpha1.Etcd) { - g.Expect(role.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(role.Labels).To(Equal(etcd.GetDefaultLabels())) - g.Expect(role.Rules).To(ConsistOf( - rbacv1.PolicyRule{ - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - )) + +func newRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { + role := emptyRole(etcd) + buildResource(etcd, role) + return role +} + +func getLatestRole(cl client.Client, etcd *druidv1alpha1.Etcd) (*rbacv1.Role, error) { + role := &rbacv1.Role{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace}, role) + return role, err +} + +func matchRole(g *WithT, etcd *druidv1alpha1.Etcd, actualRole rbacv1.Role) { + g.Expect(actualRole).To(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(etcd.GetRoleName()), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), + }), + "Rules": ConsistOf( + rbacv1.PolicyRule{ + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"apps"}, + Resources: []string{"statefulsets"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + ), + })) } diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index e69ba11ed..eec1c1496 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -56,20 +56,7 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { rb := emptyRoleBinding(etcd) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { - rb.Labels = etcd.GetDefaultLabels() - rb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - rb.RoleRef = rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: etcd.GetRoleName(), - } - rb.Subjects = []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: etcd.GetServiceAccountName(), - Namespace: etcd.Namespace, - }, - } + buildResource(etcd, rb) return nil }) if err == nil { @@ -111,3 +98,20 @@ func emptyRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { }, } } + +func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { + rb.Labels = etcd.GetDefaultLabels() + rb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + rb.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: etcd.GetRoleName(), + } + rb.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: etcd.GetServiceAccountName(), + Namespace: etcd.Namespace, + }, + } +} diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index bbaf392ce..37154650d 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -15,10 +15,10 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" ) const ( @@ -74,7 +74,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithGetError(tc.getErr) } if tc.roleBindingExists { - fakeClientBuilder.WithObjects(testsample.NewRoleBinding(etcd)) + fakeClientBuilder.WithObjects(newRoleBinding(etcd)) } operator := New(fakeClientBuilder.Build()) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -94,30 +94,16 @@ func TestSync(t *testing.T) { etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { - name string - roleBindingExists bool - getErr *apierrors.StatusError - createErr *apierrors.StatusError - expectedErr *druiderr.DruidError + name string + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { - name: "create role when none exists", - roleBindingExists: false, - }, - { - name: "create role fails when client create fails", - roleBindingExists: false, - createErr: internalStatusErr, - expectedErr: &druiderr.DruidError{ - Code: ErrSyncRoleBinding, - Cause: internalStatusErr, - Operation: "Sync", - }, + name: "create role when none exists", }, { - name: "create role fails when get errors out", - roleBindingExists: false, - getErr: internalStatusErr, + name: "create role fails when client create fails", + createErr: internalStatusErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncRoleBinding, Cause: internalStatusErr, @@ -131,29 +117,19 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder(). - WithGetError(tc.getErr). - WithCreateError(tc.createErr) - if tc.roleBindingExists { - fakeClientBuilder.WithObjects(testsample.NewRoleBinding(etcd)) - } - cl := fakeClientBuilder.Build() + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.Sync(opCtx, etcd) + syncErr := operator.Sync(opCtx, etcd) + latestRoleBinding, getErr := getLatestRoleBinding(cl, etcd) if tc.expectedErr != nil { - testutils.CheckDruidError(g, tc.expectedErr, err) + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } else { - g.Expect(err).ToNot(HaveOccurred()) - existingRoleBinding := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetRoleBindingName(), - Namespace: etcd.Namespace, - }, - } - err = cl.Get(context.Background(), client.ObjectKeyFromObject(existingRoleBinding), existingRoleBinding) - g.Expect(err).ToNot(HaveOccurred()) - checkRoleBinding(g, existingRoleBinding, etcd) + g.Expect(syncErr).ToNot(HaveOccurred()) + g.Expect(getErr).To(BeNil()) + g.Expect(latestRoleBinding).ToNot(BeNil()) + matchRoleBinding(g, etcd, *latestRoleBinding) } }) } @@ -164,19 +140,19 @@ func TestTriggerDelete(t *testing.T) { etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { - name string - roleExists bool - deleteErr *apierrors.StatusError - expectedErr *druiderr.DruidError + name string + roleBindingExists bool + deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { - name: "successfully delete existing role", - roleExists: true, + name: "successfully delete existing role", + roleBindingExists: true, }, { - name: "delete fails due to failing client delete", - roleExists: true, - deleteErr: internalStatusErr, + name: "delete fails due to failing client delete", + roleBindingExists: true, + deleteErr: internalStatusErr, expectedErr: &druiderr.DruidError{ Code: ErrDeleteRoleBinding, Cause: internalStatusErr, @@ -184,8 +160,8 @@ func TestTriggerDelete(t *testing.T) { }, }, { - name: "delete is a no-op if role does not exist", - roleExists: false, + name: "delete is a no-op if role does not exist", + roleBindingExists: false, }, } @@ -198,8 +174,8 @@ func TestTriggerDelete(t *testing.T) { if tc.deleteErr != nil { fakeClientBuilder.WithDeleteError(tc.deleteErr) } - if tc.roleExists { - fakeClientBuilder.WithObjects(testsample.NewRoleBinding(etcd)) + if tc.roleBindingExists { + fakeClientBuilder.WithObjects(newRoleBinding(etcd)) } cl := fakeClientBuilder.Build() operator := New(cl) @@ -219,19 +195,38 @@ func TestTriggerDelete(t *testing.T) { } // ---------------------------- Helper Functions ----------------------------- -func checkRoleBinding(g *WithT, roleBinding *rbacv1.RoleBinding, etcd *druidv1alpha1.Etcd) { - g.Expect(roleBinding.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(roleBinding.Labels).To(Equal(etcd.GetDefaultLabels())) - g.Expect(roleBinding.RoleRef).To(Equal(rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: etcd.GetRoleName(), + +func newRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { + rb := emptyRoleBinding(etcd) + buildResource(etcd, rb) + return rb +} + +func getLatestRoleBinding(cl client.Client, etcd *druidv1alpha1.Etcd) (*rbacv1.RoleBinding, error) { + rb := &rbacv1.RoleBinding{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace}, rb) + return rb, err +} + +func matchRoleBinding(g *WithT, etcd *druidv1alpha1.Etcd, actualRoleBinding rbacv1.RoleBinding) { + g.Expect(actualRoleBinding).To(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(etcd.GetRoleBindingName()), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), + }), + "RoleRef": Equal(rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: etcd.GetRoleName(), + }), + "Subjects": ConsistOf( + rbacv1.Subject{ + Kind: "ServiceAccount", + Name: etcd.GetServiceAccountName(), + Namespace: etcd.Namespace, + }, + ), })) - g.Expect(roleBinding.Subjects).To(ConsistOf( - rbacv1.Subject{ - Kind: "ServiceAccount", - Name: etcd.GetServiceAccountName(), - Namespace: etcd.Namespace, - }, - )) } diff --git a/test/sample/role.go b/test/sample/role.go deleted file mode 100644 index 46757c1b3..000000000 --- a/test/sample/role.go +++ /dev/null @@ -1,39 +0,0 @@ -package sample - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// NewRole creates a new sample Role initializing it from the passed in etcd object. -func NewRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { - return &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetRoleName(), - Namespace: etcd.Namespace, - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - Rules: createPolicyRules(), - } -} - -func createPolicyRules() []rbacv1.PolicyRule { - return []rbacv1.PolicyRule{ - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - } -} diff --git a/test/sample/rolebinding.go b/test/sample/rolebinding.go deleted file mode 100644 index 4165fdced..000000000 --- a/test/sample/rolebinding.go +++ /dev/null @@ -1,30 +0,0 @@ -package sample - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// NewRoleBinding creates a new RoleBinding using the passed in etcd object. -func NewRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetRoleBindingName(), - Namespace: etcd.Namespace, - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: etcd.GetServiceAccountName(), - Namespace: etcd.Namespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: etcd.GetRoleName(), - }, - } -} From ad9646078bbc612421c2068f63064f559ffee8dc Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 15 Jan 2024 14:59:05 +0530 Subject: [PATCH 073/235] removed sample pdb and adjusted tests --- .../poddisruptionbudget.go | 24 +++++---- .../poddisruptionbudget_test.go | 53 +++++++++++++------ test/sample/pdb.go | 42 --------------- 3 files changed, 50 insertions(+), 69 deletions(-) delete mode 100644 test/sample/pdb.go diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 6b3c7813b..6f0949374 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -44,16 +44,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { pdb := emptyPodDisruptionBudget(getObjectKey(etcd)) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { - pdb.Labels = etcd.GetDefaultLabels() - pdb.Annotations = getAnnotations(etcd) - pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - pdb.Spec.MinAvailable = &intstr.IntOrString{ - IntVal: computePDBMinAvailable(int(etcd.Spec.Replicas)), - Type: intstr.Int, - } - pdb.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: etcd.GetDefaultLabels(), - } + buildResource(etcd, pdb) return nil }) if err == nil { @@ -85,6 +76,19 @@ func New(client client.Client) resource.Operator { } } +func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) { + pdb.Labels = etcd.GetDefaultLabels() + pdb.Annotations = getAnnotations(etcd) + pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + pdb.Spec.MinAvailable = &intstr.IntOrString{ + IntVal: computePDBMinAvailable(int(etcd.Spec.Replicas)), + Type: intstr.Int, + } + pdb.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: etcd.GetDefaultLabels(), + } +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace} } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 0d3114d61..3d5cd400e 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -17,10 +17,11 @@ import ( corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" ) const ( @@ -73,7 +74,7 @@ func TestGetExistingResourceNames(t *testing.T) { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.pdbExists { - fakeClientBuilder.WithObjects(testsample.NewPodDisruptionBudget(etcd)) + fakeClientBuilder.WithObjects(newPodDisruptionBudget(etcd)) } operator := New(fakeClientBuilder.Build()) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -135,7 +136,8 @@ func TestSyncWhenNoPDBExists(t *testing.T) { } else { g.Expect(syncErr).ToNot(HaveOccurred()) g.Expect(getErr).ToNot(HaveOccurred()) - checkPodDisruptionBudget(g, etcd, latestPDB, tc.expectedPDBMinAvailable) + g.Expect(latestPDB).ToNot(BeNil()) + matchPodDisruptionBudget(g, etcd, *latestPDB, tc.expectedPDBMinAvailable) } }) } @@ -178,7 +180,7 @@ func TestSyncWhenPDBExists(t *testing.T) { existingEtcd := etcdBuilder.WithReplicas(tc.originalEtcdReplicas).Build() cl := testutils.NewFakeClientBuilder(). WithPatchError(tc.patchErr). - WithObjects(testsample.NewPodDisruptionBudget(existingEtcd)). + WithObjects(newPodDisruptionBudget(existingEtcd)). Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -231,7 +233,7 @@ func TestTriggerDelete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.pdbExists { - fakeClientBuilder.WithObjects(testsample.NewPodDisruptionBudget(etcd)) + fakeClientBuilder.WithObjects(newPodDisruptionBudget(etcd)) } cl := fakeClientBuilder.Build() operator := New(cl) @@ -251,18 +253,10 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- -func checkPodDisruptionBudget(g *WithT, etcd *druidv1alpha1.Etcd, actualPDB *policyv1.PodDisruptionBudget, expectedPDBMinAvailable int32) { - g.Expect(actualPDB.Name).To(Equal(etcd.Name)) - g.Expect(actualPDB.Namespace).To(Equal(etcd.Namespace)) - g.Expect(actualPDB.Labels).To(Equal(etcd.GetDefaultLabels())) - expectedAnnotations := map[string]string{ - common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - common.GardenerOwnerType: "etcd", - } - g.Expect(actualPDB.Annotations).To(Equal(expectedAnnotations)) - g.Expect(actualPDB.OwnerReferences).To(Equal([]metav1.OwnerReference{etcd.GetAsOwnerReference()})) - g.Expect(*actualPDB.Spec.Selector).To(Equal(metav1.LabelSelector{MatchLabels: etcd.GetDefaultLabels()})) - g.Expect(actualPDB.Spec.MinAvailable.IntVal).To(Equal(expectedPDBMinAvailable)) +func newPodDisruptionBudget(etcd *druidv1alpha1.Etcd) *policyv1.PodDisruptionBudget { + pdb := emptyPodDisruptionBudget(getObjectKey(etcd)) + buildResource(etcd, pdb) + return pdb } func getLatestPodDisruptionBudget(cl client.Client, etcd *druidv1alpha1.Etcd) (*policyv1.PodDisruptionBudget, error) { @@ -270,3 +264,28 @@ func getLatestPodDisruptionBudget(cl client.Client, etcd *druidv1alpha1.Etcd) (* err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace}, pdb) return pdb, err } + +func matchPodDisruptionBudget(g *WithT, etcd *druidv1alpha1.Etcd, actualPDB policyv1.PodDisruptionBudget, expectedPDBMinAvailable int32) { + expectedAnnotations := map[string]string{ + common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), + common.GardenerOwnerType: "etcd", + } + + g.Expect(actualPDB).To(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(etcd.Name), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "Annotations": testutils.MatchResourceAnnotations(expectedAnnotations), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), + }), + "Spec": MatchFields(IgnoreExtras, Fields{ + "Selector": testutils.MatchSpecLabelSelector(etcd.GetDefaultLabels()), + "MinAvailable": PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(intstr.Int), + "IntVal": Equal(expectedPDBMinAvailable), + })), + }), + })) + +} diff --git a/test/sample/pdb.go b/test/sample/pdb.go deleted file mode 100644 index 99544fb18..000000000 --- a/test/sample/pdb.go +++ /dev/null @@ -1,42 +0,0 @@ -package sample - -import ( - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - policyv1 "k8s.io/api/policy/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// NewPodDisruptionBudget creates a new instance of PodDisruptionBudget from the passed etcd object. -func NewPodDisruptionBudget(etcd *druidv1alpha1.Etcd) *policyv1.PodDisruptionBudget { - return &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.Name, - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - Annotations: map[string]string{ - "gardener.cloud/owned-by": fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - "gardener.cloud/owner-type": "etcd", - }, - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: etcd.GetDefaultLabels(), - }, - MinAvailable: &intstr.IntOrString{ - IntVal: computePDBMinAvailable(etcd.Spec.Replicas), - Type: intstr.Int, - }, - }, - } -} - -func computePDBMinAvailable(etcdReplicas int32) int32 { - if etcdReplicas <= 1 { - return 0 - } - return etcdReplicas/2 + 1 -} From 5e1e4761f93b6aef34f14f823ec7f52e71796845 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 15 Jan 2024 18:40:09 +0530 Subject: [PATCH 074/235] removed sample sa and adjusted tests --- .../operator/serviceaccount/serviceaccount.go | 10 +++-- .../serviceaccount/serviceaccount_test.go | 41 +++++++++---------- test/sample/serviceaccount.go | 21 ---------- test/utils/matcher.go | 6 +++ 4 files changed, 33 insertions(+), 45 deletions(-) delete mode 100644 test/sample/serviceaccount.go diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 1f6ed5315..7f7eaa54b 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -45,9 +45,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { sa := emptyServiceAccount(getObjectKey(etcd)) opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, sa, func() error { - sa.Labels = etcd.GetDefaultLabels() - sa.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - sa.AutomountServiceAccountToken = pointer.Bool(!r.disableAutoMount) + buildResource(etcd, sa, !r.disableAutoMount) return nil }) if err == nil { @@ -80,6 +78,12 @@ func New(client client.Client, disableAutomount bool) resource.Operator { } } +func buildResource(etcd *druidv1alpha1.Etcd, sa *corev1.ServiceAccount, autoMountServiceAccountToken bool) { + sa.Labels = etcd.GetDefaultLabels() + sa.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + sa.AutomountServiceAccountToken = pointer.Bool(autoMountServiceAccountToken) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace} } diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index 3d68aeb11..586f8fe71 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -12,10 +12,8 @@ import ( testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" - gomegatypes "github.com/onsi/gomega/types" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/gomega" @@ -71,7 +69,7 @@ func TestGetExistingResourceNames(t *testing.T) { t.Run(tc.name, func(t *testing.T) { fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.saExists { - fakeClientBuilder.WithObjects(testsample.NewServiceAccount(etcd, false)) + fakeClientBuilder.WithObjects(newServiceAccount(etcd, false)) } cl := fakeClientBuilder.Build() operator := New(cl, true) @@ -81,8 +79,8 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { if tc.saExists { - existingSA, err := getLatestServiceAccount(cl, etcd) - g.Expect(err).ToNot(HaveOccurred()) + existingSA, getErr := getLatestServiceAccount(cl, etcd) + g.Expect(getErr).ToNot(HaveOccurred()) g.Expect(saNames).To(HaveLen(1)) g.Expect(saNames[0]).To(Equal(existingSA.Name)) } else { @@ -134,7 +132,8 @@ func TestSync(t *testing.T) { g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } else { g.Expect(getErr).To(BeNil()) - g.Expect(*latestSA).To(matchServiceAccount(etcd.GetServiceAccountName(), etcd.Namespace, etcd.Name, etcd.UID, tc.disableAutoMount)) + g.Expect(latestSA).ToNot(BeNil()) + matchServiceAccount(g, etcd, *latestSA, tc.disableAutoMount) } }) } @@ -175,7 +174,7 @@ func TestTriggerDelete(t *testing.T) { etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.saExists { - fakeClientBuilder.WithObjects(testsample.NewServiceAccount(etcd, false)) + fakeClientBuilder.WithObjects(newServiceAccount(etcd, false)) } cl := fakeClientBuilder.Build() operator := New(cl, false) @@ -195,26 +194,26 @@ func TestTriggerDelete(t *testing.T) { } // ---------------------------- Helper Functions ----------------------------- +func newServiceAccount(etcd *druidv1alpha1.Etcd, disableAutomount bool) *corev1.ServiceAccount { + sa := emptyServiceAccount(getObjectKey(etcd)) + buildResource(etcd, sa, !disableAutomount) + return sa +} + func getLatestServiceAccount(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.ServiceAccount, error) { sa := &corev1.ServiceAccount{} err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace}, sa) return sa, err } -func matchServiceAccount(saName, saNamespace, etcdName string, etcdUID types.UID, disableAutomount bool) gomegatypes.GomegaMatcher { - return MatchFields(IgnoreExtras, Fields{ +func matchServiceAccount(g *WithT, etcd *druidv1alpha1.Etcd, actualSA corev1.ServiceAccount, disableAutoMount bool) { + g.Expect(actualSA).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(saName), - "Namespace": Equal(saNamespace), - "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(etcdName), - "UID": Equal(etcdUID), - "Controller": PointTo(BeTrue()), - "BlockOwnerDeletion": PointTo(BeTrue()), - })), + "Name": Equal(etcd.GetServiceAccountName()), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), - "AutomountServiceAccountToken": PointTo(Equal(!disableAutomount)), - }) + "AutomountServiceAccountToken": PointTo(Equal(!disableAutoMount)), + })) } diff --git a/test/sample/serviceaccount.go b/test/sample/serviceaccount.go deleted file mode 100644 index aaa683f73..000000000 --- a/test/sample/serviceaccount.go +++ /dev/null @@ -1,21 +0,0 @@ -package sample - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" -) - -// NewServiceAccount creates a new sample ServiceAccount using the passed in etcd. -func NewServiceAccount(etcd *druidv1alpha1.Etcd, disableAutoMount bool) *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetServiceAccountName(), - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - AutomountServiceAccountToken: pointer.Bool(!disableAutoMount), - } -} diff --git a/test/utils/matcher.go b/test/utils/matcher.go index 6f14084a1..bee525986 100644 --- a/test/utils/matcher.go +++ b/test/utils/matcher.go @@ -74,6 +74,12 @@ func MatchResourceLabels(expected map[string]string) gomegatypes.GomegaMatcher { } } +func MatchSpecLabelSelector(expected map[string]string) gomegatypes.GomegaMatcher { + return PointTo(MatchFields(IgnoreExtras, Fields{ + "MatchLabels": MatchResourceLabels(expected), + })) +} + // MatchEtcdOwnerReference is a custom gomega matcher which creates a matcher for ObjectMeta.OwnerReferences func MatchEtcdOwnerReference(etcdName string, etcdUID types.UID) gomegatypes.GomegaMatcher { return ConsistOf(MatchFields(IgnoreExtras, Fields{ From a78fa686eee38cede0399e095913dbd5ee42d0a1 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 16 Jan 2024 11:58:11 +0530 Subject: [PATCH 075/235] removed sample for leases and adapted tests --- internal/operator/memberlease/memberlease.go | 8 +- .../operator/memberlease/memberlease_test.go | 100 ++++++++++-------- .../operator/snapshotlease/snapshotlease.go | 18 ++-- .../snapshotlease/snapshotlease_test.go | 87 +++++++++------ .../controllers/compaction/reconciler_test.go | 4 +- test/sample/etcd.go | 4 +- test/sample/memberlease.go | 30 ------ test/sample/snapshotlease.go | 35 ------ test/utils/lease.go | 9 +- 9 files changed, 135 insertions(+), 160 deletions(-) delete mode 100644 test/sample/memberlease.go delete mode 100644 test/sample/snapshotlease.go diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 7768b38dd..eaf69b341 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -73,8 +73,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { lease := emptyMemberLease(objKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { - lease.Labels = utils.MergeMaps[string](utils.GetMemberLeaseLabels(etcd.Name), etcd.GetDefaultLabels()) - lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + buildResource(etcd, lease) return nil }) if err != nil { @@ -108,6 +107,11 @@ func New(client client.Client) resource.Operator { } } +func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { + lease.Labels = utils.MergeMaps[string](utils.GetMemberLeaseLabels(etcd.Name), etcd.GetDefaultLabels()) + lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} +} + func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { leaseNames := etcd.GetMemberLeaseNames() objectKeys := make([]client.ObjectKey, 0, len(leaseNames)) diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 6e0d9ed79..1054483d6 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -21,13 +21,15 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" + testEtcdName = "test-etcd" + nonTargetEtcdName = "another-etcd" + testNs = "test-namespace" ) var ( @@ -88,7 +90,7 @@ func TestGetExistingResourceNames(t *testing.T) { WithGetError(tc.getErr). WithListError(tc.listErr) if tc.numExistingLeases > 0 { - leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) + leases, err := newMemberLeases(etcd, tc.numExistingLeases) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { fakeClientBuilder.WithObjects(lease) @@ -163,7 +165,7 @@ func TestSync(t *testing.T) { etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).WithGetError(tc.getErr) if tc.numExistingLeases > 0 { - leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) + leases, err := newMemberLeases(etcd, tc.numExistingLeases) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { fakeClientBuilder.WithObjects(lease) @@ -197,12 +199,11 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { testCases := []struct { - name string - etcdReplicas int32 // original replicas - numExistingLeases int - testWithSnapshotLeases bool // this is to ensure that delete of member leases should not delete snapshot leases - deleteAllOfErr *apierrors.StatusError - expectedErr *druiderr.DruidError + name string + etcdReplicas int32 // original replicas + numExistingLeases int + deleteAllOfErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { name: "no-op when no member lease exists", @@ -215,10 +216,9 @@ func TestTriggerDelete(t *testing.T) { numExistingLeases: 3, }, { - name: "only delete member leases and not snapshot leases", - etcdReplicas: 3, - numExistingLeases: 3, - testWithSnapshotLeases: true, + name: "only delete member leases and not snapshot leases", + etcdReplicas: 3, + numExistingLeases: 3, }, { name: "successfully deletes remainder member leases", @@ -240,20 +240,22 @@ func TestTriggerDelete(t *testing.T) { g := NewWithT(t) t.Parallel() + nonTargetEtcd := testsample.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() + nonTargetLeaseNames := []string{"another-etcd-0", "another-etcd-1"} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // *************** set up existing environment ***************** etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).WithReplicas(tc.etcdReplicas).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllOfErr) if tc.numExistingLeases > 0 { - leases, err := testsample.NewMemberLeases(etcd, tc.numExistingLeases, getAdditionalMemberLeaseLabels(etcd.Name)) + leases, err := newMemberLeases(etcd, tc.numExistingLeases) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { fakeClientBuilder.WithObjects(lease) } } - if tc.testWithSnapshotLeases { - fakeClientBuilder.WithObjects(testsample.NewDeltaSnapshotLease(etcd), testsample.NewFullSnapshotLease(etcd)) + for _, nonTargetLeaseName := range nonTargetLeaseNames { + fakeClientBuilder.WithObjects(testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, purpose)) } cl := fakeClientBuilder.Build() // ***************** Setup operator and test ***************** @@ -268,23 +270,15 @@ func TestTriggerDelete(t *testing.T) { g.Expect(memberLeasesPostDelete).Should(HaveLen(tc.numExistingLeases)) } else { g.Expect(memberLeasesPostDelete).Should(HaveLen(0)) - if tc.testWithSnapshotLeases { - snapshotLeases := getLatestSnapshotLeases(g, cl, etcd) - g.Expect(snapshotLeases).To(HaveLen(2)) - } + nonTargetMemberLeases := getLatestNonTargetMemberLeases(g, cl, nonTargetEtcd) + g.Expect(nonTargetMemberLeases).To(HaveLen(len(nonTargetLeaseNames))) + g.Expect(nonTargetMemberLeases).To(ConsistOf(memberLeases(nonTargetEtcd.Name, nonTargetEtcd.UID, int32(len(nonTargetLeaseNames))))) } }) } } // ---------------------------- Helper Functions ----------------------------- -func getAdditionalMemberLeaseLabels(etcdName string) map[string]string { - return map[string]string{ - common.GardenerOwnedBy: etcdName, - v1beta1constants.GardenerPurpose: purpose, - } -} - func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { return doGetLatestLeases(g, cl, etcd, map[string]string{ common.GardenerOwnedBy: etcd.Name, @@ -292,11 +286,8 @@ func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) }) } -func getLatestSnapshotLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { - return doGetLatestLeases(g, cl, etcd, map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", - }) +func getLatestNonTargetMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { + return doGetLatestLeases(g, cl, etcd, getAdditionalMemberLeaseLabels(etcd.Name)) } func doGetLatestLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) []coordinationv1.Lease { @@ -308,9 +299,9 @@ func doGetLatestLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd, mat return leases.Items } -func memberLeases(etcdName string, etcdUID types.UID, replicas int32) []interface{} { +func memberLeases(etcdName string, etcdUID types.UID, numLeases int32) []interface{} { var elements []interface{} - for i := 0; i < int(replicas); i++ { + for i := 0; i < int(numLeases); i++ { leaseName := fmt.Sprintf("%s-%d", etcdName, i) elements = append(elements, matchLeaseElement(leaseName, etcdName, etcdUID)) } @@ -320,15 +311,36 @@ func memberLeases(etcdName string, etcdUID types.UID, replicas int32) []interfac func matchLeaseElement(leaseName, etcdName string, etcdUID types.UID) gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(leaseName), - "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(etcdName), - "UID": Equal(etcdUID), - "Controller": PointTo(BeTrue()), - "BlockOwnerDeletion": PointTo(BeTrue()), - })), + "Name": Equal(leaseName), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcdName, etcdUID), }), }) } + +func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1.Lease, error) { + if numLeases > int(etcd.Spec.Replicas) { + return nil, errors.New("number of requested leases is greater than the etcd replicas") + } + additionalLabels := getAdditionalMemberLeaseLabels(etcd.Name) + memberLeaseNames := etcd.GetMemberLeaseNames() + leases := make([]*coordinationv1.Lease, 0, numLeases) + for i := 0; i < numLeases; i++ { + lease := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberLeaseNames[i], + Namespace: etcd.Namespace, + Labels: testutils.MergeMaps[string, string](etcd.GetDefaultLabels(), additionalLabels), + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + } + leases = append(leases, lease) + } + return leases, nil +} + +func getAdditionalMemberLeaseLabels(etcdName string) map[string]string { + return map[string]string{ + common.GardenerOwnedBy: etcdName, + v1beta1constants.GardenerPurpose: purpose, + } +} diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 6c839a663..b6b93ecd1 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -77,12 +77,12 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } objectKeys := getObjectKeys(etcd) - createTasks := make([]utils.OperatorTask, len(objectKeys)) + syncTasks := make([]utils.OperatorTask, len(objectKeys)) var errs error for i, objKey := range objectKeys { objKey := objKey // capture the range variable - createTasks[i] = utils.OperatorTask{ + syncTasks[i] = utils.OperatorTask{ Name: "CreateOrUpdate-" + objKey.String(), Fn: func(ctx resource.OperatorContext) error { return r.doCreateOrUpdate(ctx, etcd, objKey) @@ -90,11 +90,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } } - if errorList := utils.RunConcurrently(ctx, createTasks); len(errorList) > 0 { - for _, err := range errorList { - errs = multierror.Append(errs, err) - } - } + errs = multierror.Append(errs, utils.RunConcurrently(ctx, syncTasks)...) return errs } @@ -142,8 +138,7 @@ func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*c func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) error { lease := emptySnapshotLease(leaseObjectKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { - lease.Labels = getLabels(etcd) - lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + buildResource(etcd, lease) return nil }) if err != nil { @@ -157,6 +152,11 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a return nil } +func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { + lease.Labels = getLabels(etcd) + lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} +} + func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { labels := make(map[string]string) labels[common.GardenerOwnedBy] = etcd.Name diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 3a9730ef2..09c09a9d4 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -21,12 +21,14 @@ import ( gomegatypes "github.com/onsi/gomega/types" coordinationv1 "k8s.io/api/coordination/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" + testEtcdName = "test-etcd" + nonTargetEtcdName = "another-etcd" + testNs = "test-namespace" ) var ( @@ -79,7 +81,7 @@ func TestGetExistingResourceNames(t *testing.T) { etcd := etcdBuilder.Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) if tc.backupEnabled { - fakeClientBuilder.WithObjects(testsample.NewDeltaSnapshotLease(etcd), testsample.NewFullSnapshotLease(etcd)) + fakeClientBuilder.WithObjects(newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) } operator := New(fakeClientBuilder.Build()) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -138,6 +140,7 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { } func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { + nonTargetEtcd := testsample.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() existingEtcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() // backup is enabled updatedEtcd := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs).Build() // backup is disabled testCases := []struct { @@ -165,7 +168,11 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cl := testutils.NewFakeClientBuilder(). WithDeleteAllOfError(tc.deleteAllOfErr). - WithObjects(testsample.NewDeltaSnapshotLease(existingEtcd), testsample.NewFullSnapshotLease(existingEtcd)). + WithObjects( + newDeltaSnapshotLease(existingEtcd), + newFullSnapshotLease(existingEtcd), + newDeltaSnapshotLease(nonTargetEtcd), + newFullSnapshotLease(nonTargetEtcd)). Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -177,6 +184,10 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { g.Expect(latestSnapshotLeases).To(HaveLen(2)) } else { g.Expect(latestSnapshotLeases).To(HaveLen(0)) + // To ensure that delete of snapshot leases did not remove non-target- snapshot leases also check that these still exist + actualNonTargetSnapshotLeases, nonTargetSnapshotListErr := getLatestNonTargetSnapshotLeases(cl, nonTargetEtcd) + g.Expect(nonTargetSnapshotListErr).To(BeNil()) + g.Expect(actualNonTargetSnapshotLeases).To(HaveLen(2)) } }) } @@ -184,6 +195,7 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { + nonTargetEtcd := testsample.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() etcdBuilder := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs).WithReplicas(3) testCases := []struct { name string @@ -219,42 +231,60 @@ func TestTriggerDelete(t *testing.T) { etcdBuilder.WithDefaultBackup() } etcd := etcdBuilder.Build() - fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllErr) - memberLeases, err := testsample.NewMemberLeases(etcd, int(etcd.Spec.Replicas), map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-member-lease", - }) - g.Expect(err).ToNot(HaveOccurred()) - for _, lease := range memberLeases { - fakeClientBuilder.WithObjects(lease) - } + fakeClientBuilder := testutils.NewFakeClientBuilder(). + WithDeleteAllOfError(tc.deleteAllErr). + WithObjects( + newDeltaSnapshotLease(nonTargetEtcd), + newFullSnapshotLease(nonTargetEtcd), + ) if tc.backupEnabled { - fakeClientBuilder.WithObjects(testsample.NewDeltaSnapshotLease(etcd), testsample.NewFullSnapshotLease(etcd)) + fakeClientBuilder.WithObjects(newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) } cl := fakeClientBuilder.Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestSnapshotLeases, snapshotLeaseListErr := getLatestSnapshotLeases(cl, etcd) - latestMemberLeases, memberLeaseListErr := getLatestMemberLeases(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) g.Expect(snapshotLeaseListErr).ToNot(HaveOccurred()) g.Expect(latestSnapshotLeases).To(HaveLen(2)) - g.Expect(memberLeaseListErr).ToNot(HaveOccurred()) - g.Expect(latestMemberLeases).To(HaveLen(int(etcd.Spec.Replicas))) } else { g.Expect(triggerDeleteErr).ToNot(HaveOccurred()) g.Expect(snapshotLeaseListErr).ToNot(HaveOccurred()) g.Expect(latestSnapshotLeases).To(HaveLen(0)) - g.Expect(memberLeaseListErr).ToNot(HaveOccurred()) - g.Expect(latestMemberLeases).To(HaveLen(int(etcd.Spec.Replicas))) } + actualNonTargetSnapshotLeases, nonTargetSnapshotListErr := getLatestNonTargetSnapshotLeases(cl, nonTargetEtcd) + g.Expect(nonTargetSnapshotListErr).ToNot(HaveOccurred()) + g.Expect(actualNonTargetSnapshotLeases).To(HaveLen(2)) }) } } // ---------------------------- Helper Functions ----------------------------- +func newDeltaSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { + leaseName := etcd.GetDeltaSnapshotLeaseName() + return buildLease(etcd, leaseName) +} + +func newFullSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { + leaseName := etcd.GetFullSnapshotLeaseName() + return buildLease(etcd, leaseName) +} + +func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: leaseName, + Namespace: etcd.Namespace, + Labels: utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + "gardener.cloud/owned-by": etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", + }), + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + } +} func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ @@ -263,25 +293,18 @@ func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMa }) return MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(leaseName), - "Namespace": Equal(etcd.Namespace), - "Labels": Equal(expectedLabels), - "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(etcd.Name), - "UID": Equal(etcd.UID), - "Controller": PointTo(BeTrue()), - "BlockOwnerDeletion": PointTo(BeTrue()), - })), + "Name": Equal(leaseName), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(expectedLabels), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), }) } -func getLatestMemberLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { +func getLatestNonTargetSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { return doGetLatestLeases(cl, etcd, map[string]string{ common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-member-lease", + v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", }) } diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index d513adb72..79453a2fa 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -732,11 +732,11 @@ func deleteEtcdAndWait(c client.Client, etcd *druidv1alpha1.Etcd) { func createEtcdSnapshotLeasesAndWait(c client.Client, etcd *druidv1alpha1.Etcd) (*coordinationv1.Lease, *coordinationv1.Lease) { By("create full snapshot lease") - fullSnapLease := testutils.CreateLease(etcd.GetFullSnapshotLeaseName(), etcd.Namespace, etcd.Name, etcd.UID) + fullSnapLease := testutils.CreateLease(etcd.GetFullSnapshotLeaseName(), etcd.Namespace, etcd.Name, etcd.UID, "etcd-snapshot-lease") Expect(c.Create(context.TODO(), fullSnapLease)).To(Succeed()) By("create delta snapshot lease") - deltaSnapLease := testutils.CreateLease(etcd.GetDeltaSnapshotLeaseName(), etcd.Namespace, etcd.Name, etcd.UID) + deltaSnapLease := testutils.CreateLease(etcd.GetDeltaSnapshotLeaseName(), etcd.Namespace, etcd.Name, etcd.UID, "etcd-snapshot-lease") Expect(c.Create(context.TODO(), deltaSnapLease)).To(Succeed()) // wait for full snapshot lease to be created diff --git a/test/sample/etcd.go b/test/sample/etcd.go index 7157cce91..f42b29a46 100644 --- a/test/sample/etcd.go +++ b/test/sample/etcd.go @@ -18,6 +18,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/google/uuid" testutils "github.com/gardener/etcd-druid/test/utils" @@ -52,7 +53,6 @@ var ( quota = resource.MustParse("8Gi") localProvider = druidv1alpha1.StorageProvider("Local") prefix = "/tmp" - uid = "a9b8c7d6e5f4" volumeClaimTemplateName = "etcd-main" garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) metricsBasic = druidv1alpha1.Basic @@ -341,7 +341,7 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, - UID: types.UID(uid), + UID: types.UID(uuid.NewString()), }, Spec: druidv1alpha1.EtcdSpec{ Annotations: map[string]string{ diff --git a/test/sample/memberlease.go b/test/sample/memberlease.go deleted file mode 100644 index 60d54e1e0..000000000 --- a/test/sample/memberlease.go +++ /dev/null @@ -1,30 +0,0 @@ -package sample - -import ( - "errors" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - testutils "github.com/gardener/etcd-druid/test/utils" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func NewMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int, additionalLabels map[string]string) ([]*coordinationv1.Lease, error) { - if numLeases > int(etcd.Spec.Replicas) { - return nil, errors.New("number of requested leases is greater than the etcd replicas") - } - memberLeaseNames := etcd.GetMemberLeaseNames() - leases := make([]*coordinationv1.Lease, 0, numLeases) - for i := 0; i < numLeases; i++ { - lease := &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: memberLeaseNames[i], - Namespace: etcd.Namespace, - Labels: testutils.MergeMaps[string, string](etcd.GetDefaultLabels(), additionalLabels), - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - } - leases = append(leases, lease) - } - return leases, nil -} diff --git a/test/sample/snapshotlease.go b/test/sample/snapshotlease.go deleted file mode 100644 index 0b7eb68e9..000000000 --- a/test/sample/snapshotlease.go +++ /dev/null @@ -1,35 +0,0 @@ -package sample - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// NewDeltaSnapshotLease creates a delta snapshot lease from the passed in etcd object. -func NewDeltaSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { - leaseName := etcd.GetDeltaSnapshotLeaseName() - return createLease(etcd, leaseName) -} - -// NewFullSnapshotLease creates a full snapshot lease from the passed in etcd object. -func NewFullSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { - leaseName := etcd.GetFullSnapshotLeaseName() - return createLease(etcd, leaseName) -} - -func createLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Lease { - return &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: leaseName, - Namespace: etcd.Namespace, - Labels: utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ - "gardener.cloud/owned-by": etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", - }), - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, - }, - } -} diff --git a/test/utils/lease.go b/test/utils/lease.go index cea4320d3..335bc00fc 100644 --- a/test/utils/lease.go +++ b/test/utils/lease.go @@ -19,14 +19,16 @@ import ( ) // CreateLease creates a lease with its owner reference set to etcd. -func CreateLease(name, namespace, etcdName string, etcdUID types.UID) *coordinationv1.Lease { +func CreateLease(name, namespace, etcdName string, etcdUID types.UID, purpose string) *coordinationv1.Lease { return &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: map[string]string{ - "instance": etcdName, - "name": "etcd", + "name": "etcd", + "instance": etcdName, + "gardener.cloud/owned-by": etcdName, + "gardener.cloud/purpose": purpose, }, OwnerReferences: []metav1.OwnerReference{{ APIVersion: druidv1alpha1.GroupVersion.String(), @@ -37,7 +39,6 @@ func CreateLease(name, namespace, etcdName string, etcdUID types.UID) *coordinat BlockOwnerDeletion: pointer.Bool(true), }}, }, - Spec: coordinationv1.LeaseSpec{}, } } From 967f39d0b8724a0f6eec47ed6542eab9dacfcfc7 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 16 Jan 2024 12:07:04 +0530 Subject: [PATCH 076/235] moved etcd builder back to test utils and adapted tests --- .../clientservice/clientservice_test.go | 7 +- .../operator/memberlease/memberlease_test.go | 9 +- .../operator/peerservice/peerservice_test.go | 9 +- .../poddisruptionbudget_test.go | 9 +- internal/operator/role/role_test.go | 7 +- .../operator/rolebinding/rolebinding_test.go | 7 +- .../serviceaccount/serviceaccount_test.go | 7 +- .../snapshotlease/snapshotlease_test.go | 15 +- test/sample/etcd.go | 448 ------------------ test/utils/etcd.go | 422 +++++++++++++++++ 10 files changed, 453 insertions(+), 487 deletions(-) delete mode 100644 test/sample/etcd.go diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 5fa9adfb0..3de337f57 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -26,7 +26,6 @@ import ( "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -50,7 +49,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string svcExists bool @@ -245,7 +244,7 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // ********************* Setup ********************* - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -271,7 +270,7 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) if clientPort != nil { etcdBuilder.WithEtcdClientPort(clientPort) } diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 1054483d6..2e54211f8 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -10,7 +10,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/go-logr/logr" @@ -39,7 +38,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string etcdReplicas int32 @@ -112,7 +111,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string etcdReplicas int32 // original replicas @@ -240,12 +239,12 @@ func TestTriggerDelete(t *testing.T) { g := NewWithT(t) t.Parallel() - nonTargetEtcd := testsample.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() + nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() nonTargetLeaseNames := []string{"another-etcd-0", "another-etcd-1"} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // *************** set up existing environment ***************** - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).WithReplicas(tc.etcdReplicas).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).WithReplicas(tc.etcdReplicas).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllOfErr) if tc.numExistingLeases > 0 { leases, err := newMemberLeases(etcd, tc.numExistingLeases) diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index aebc564a3..94d6e46ea 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -23,7 +23,6 @@ import ( druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -48,7 +47,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testcases := []struct { name string svcExists bool @@ -103,7 +102,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenNoServiceExists(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string createWithPort *int32 @@ -154,7 +153,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { } func TestSyncWhenServiceExists(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string updateWithPort *int32 @@ -203,7 +202,7 @@ func TestSyncWhenServiceExists(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestPeerServiceTriggerDelete(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() deleteInternalErr := apierrors.NewInternalError(errors.New("fake delete internal error")) testCases := []struct { name string diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 3d5cd400e..0661f43d0 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -10,7 +10,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -36,7 +35,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string pdbExists bool @@ -91,7 +90,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenNoPDBExists(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string etcdReplicas int32 @@ -144,7 +143,7 @@ func TestSyncWhenNoPDBExists(t *testing.T) { } func TestSyncWhenPDBExists(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) testCases := []struct { name string originalEtcdReplicas int32 @@ -200,7 +199,7 @@ func TestSyncWhenPDBExists(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string pdbExists bool diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 03e250ff7..07cf0b0cb 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -8,7 +8,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -31,7 +30,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() getInternalErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -90,7 +89,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -138,7 +137,7 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index 37154650d..32f71ab85 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -8,7 +8,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -32,7 +31,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() getErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -91,7 +90,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -137,7 +136,7 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index 586f8fe71..cd903b93f 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -8,7 +8,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -32,7 +31,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string saExists bool @@ -122,7 +121,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() operator := New(cl, tc.disableAutoMount) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -171,7 +170,7 @@ func TestTriggerDelete(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.saExists { fakeClientBuilder.WithObjects(newServiceAccount(etcd, false)) diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 09c09a9d4..5e49e9b49 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -11,7 +11,6 @@ import ( druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/go-logr/logr" @@ -38,7 +37,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcdBuilder := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testEtcdName, testNs) testCases := []struct { name string backupEnabled bool @@ -98,7 +97,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenBackupIsEnabled(t *testing.T) { - etcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() testCases := []struct { name string createErr *apierrors.StatusError @@ -140,9 +139,9 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { } func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { - nonTargetEtcd := testsample.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() - existingEtcd := testsample.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() // backup is enabled - updatedEtcd := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs).Build() // backup is disabled + nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() + existingEtcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() // backup is enabled + updatedEtcd := testutils.EtcdBuilderWithoutDefaults(testEtcdName, testNs).Build() // backup is disabled testCases := []struct { name string deleteAllOfErr *apierrors.StatusError @@ -195,8 +194,8 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - nonTargetEtcd := testsample.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() - etcdBuilder := testsample.EtcdBuilderWithoutDefaults(testEtcdName, testNs).WithReplicas(3) + nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() + etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testEtcdName, testNs).WithReplicas(3) testCases := []struct { name string backupEnabled bool diff --git a/test/sample/etcd.go b/test/sample/etcd.go deleted file mode 100644 index f42b29a46..000000000 --- a/test/sample/etcd.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sample - -import ( - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/google/uuid" - - testutils "github.com/gardener/etcd-druid/test/utils" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" -) - -var ( - deltaSnapshotPeriod = metav1.Duration{ - Duration: 300 * time.Second, - } - garbageCollectionPeriod = metav1.Duration{ - Duration: 43200 * time.Second, - } - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - imageEtcd = "eu.gcr.io/gardener-project/gardener/etcd-wrapper:v0.1.0" - imageBR = "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.25.0" - snapshotSchedule = "0 */24 * * *" - defragSchedule = "0 */24 * * *" - container = "default.bkp" - storageCapacity = resource.MustParse("5Gi") - storageClass = "default" - priorityClassName = "class_priority" - deltaSnapShotMemLimit = resource.MustParse("100Mi") - autoCompactionMode = druidv1alpha1.Periodic - autoCompactionRetention = "2m" - quota = resource.MustParse("8Gi") - localProvider = druidv1alpha1.StorageProvider("Local") - prefix = "/tmp" - volumeClaimTemplateName = "etcd-main" - garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) - metricsBasic = druidv1alpha1.Basic - etcdSnapshotTimeout = metav1.Duration{ - Duration: 10 * time.Minute, - } - etcdDefragTimeout = metav1.Duration{ - Duration: 10 * time.Minute, - } -) - -type EtcdBuilder struct { - etcd *druidv1alpha1.Etcd -} - -func EtcdBuilderWithDefaults(name, namespace string) *EtcdBuilder { - builder := EtcdBuilder{} - builder.etcd = getDefaultEtcd(name, namespace) - return &builder -} - -func EtcdBuilderWithoutDefaults(name, namespace string) *EtcdBuilder { - builder := EtcdBuilder{} - builder.etcd = getEtcdWithoutDefaults(name, namespace) - return &builder -} - -func (eb *EtcdBuilder) WithReplicas(replicas int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Replicas = replicas - return eb -} - -func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - clientTLSConfig := &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "client-url-ca-etcd", - }, - DataKey: pointer.String("ca.crt"), - }, - ClientTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-client-tls", - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-server-tls", - }, - } - - peerTLSConfig := &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "peer-url-ca-etcd", - }, - DataKey: pointer.String("ca.crt"), - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "peer-url-etcd-server-tls", - }, - } - - eb.etcd.Spec.Etcd.ClientUrlTLS = clientTLSConfig - eb.etcd.Spec.Etcd.PeerUrlTLS = peerTLSConfig - eb.etcd.Spec.Backup.TLS = clientTLSConfig - - return eb -} - -func (eb *EtcdBuilder) WithReadyStatus() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - - members := make([]druidv1alpha1.EtcdMemberStatus, 0) - for i := 0; i < int(eb.etcd.Spec.Replicas); i++ { - members = append(members, druidv1alpha1.EtcdMemberStatus{Status: druidv1alpha1.EtcdMemberStatusReady}) - } - eb.etcd.Status = druidv1alpha1.EtcdStatus{ - ReadyReplicas: eb.etcd.Spec.Replicas, - Replicas: eb.etcd.Spec.Replicas, - CurrentReplicas: eb.etcd.Spec.Replicas, - UpdatedReplicas: eb.etcd.Spec.Replicas, - Ready: pointer.Bool(true), - Members: members, - Conditions: []druidv1alpha1.Condition{ - {Type: druidv1alpha1.ConditionTypeAllMembersReady, Status: druidv1alpha1.ConditionTrue}, - }, - } - - return eb -} - -func (eb *EtcdBuilder) WithStorageProvider(provider druidv1alpha1.StorageProvider) *EtcdBuilder { - // TODO: there is no default case right now which is not very right, returning an error in a default case makes it difficult to chain - // This should be improved later - switch provider { - case "aws": - return eb.WithProviderS3() - case "azure": - return eb.WithProviderABS() - case "alicloud": - return eb.WithProviderOSS() - case "gcp": - return eb.WithProviderGCS() - case "openstack": - return eb.WithProviderSwift() - case "local": - return eb.WithProviderLocal() - default: - return eb - } -} - -func (eb *EtcdBuilder) WithProviderS3() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "aws", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderABS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "azure", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderGCS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "gcp", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderSwift() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "openstack", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderOSS() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStore( - eb.etcd.Name, - "alicloud", - ) - return eb -} - -func (eb *EtcdBuilder) WithProviderLocal() *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Store = getBackupStoreForLocal( - eb.etcd.Name, - ) - return eb -} - -func (eb *EtcdBuilder) WithEtcdClientPort(clientPort *int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Etcd.ClientPort = clientPort - return eb -} - -func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - - if eb.etcd.Spec.Etcd.ClientService == nil { - eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} - } - - eb.etcd.Spec.Etcd.ClientService.Labels = testutils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) - return eb -} - -func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]string) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - - if eb.etcd.Spec.Etcd.ClientService == nil { - eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} - } - - eb.etcd.Spec.Etcd.ClientService.Annotations = testutils.MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) - return eb -} - -func (eb *EtcdBuilder) WithEtcdServerPort(serverPort *int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Etcd.ServerPort = serverPort - return eb -} - -func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { - if eb == nil || eb.etcd == nil { - return nil - } - eb.etcd.Spec.Backup.Port = port - return eb -} - -// WithLabels merges the labels that already exists with the ones that are passed to this method. If an entry is -// already present in the existing labels then it gets overwritten with the new value present in the passed in labels. -func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { - testutils.MergeMaps[string, string](eb.etcd.Labels, labels) - return eb -} - -// WithDefaultBackup creates a default backup spec and initializes etcd with it. -func (eb *EtcdBuilder) WithDefaultBackup() *EtcdBuilder { - eb.etcd.Spec.Backup = getBackupSpec() - return eb -} - -func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { - return eb.etcd -} - -func getEtcdWithoutDefaults(name, namespace string) *druidv1alpha1.Etcd { - return &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: druidv1alpha1.EtcdSpec{ - Annotations: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - "role": "test", - }, - Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - }, - Replicas: 1, - Backup: druidv1alpha1.BackupSpec{}, - Etcd: druidv1alpha1.EtcdConfig{}, - Common: druidv1alpha1.SharedConfig{}, - }, - } -} - -func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { - return &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: types.UID(uuid.NewString()), - }, - Spec: druidv1alpha1.EtcdSpec{ - Annotations: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - "role": "test", - }, - Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", - }, - }, - Replicas: 1, - StorageCapacity: &storageCapacity, - StorageClass: &storageClass, - PriorityClassName: &priorityClassName, - VolumeClaimTemplate: &volumeClaimTemplateName, - Backup: getBackupSpec(), - Etcd: druidv1alpha1.EtcdConfig{ - Quota: "a, - Metrics: &metricsBasic, - Image: &imageEtcd, - DefragmentationSchedule: &defragSchedule, - EtcdDefragTimeout: &etcdDefragTimeout, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("2500m"), - "memory": testutils.ParseQuantity("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("1000Mi"), - }, - }, - ClientPort: &clientPort, - ServerPort: &serverPort, - }, - Common: druidv1alpha1.SharedConfig{ - AutoCompactionMode: &autoCompactionMode, - AutoCompactionRetention: &autoCompactionRetention, - }, - }, - } -} - -func getBackupSpec() druidv1alpha1.BackupSpec { - return druidv1alpha1.BackupSpec{ - Image: &imageBR, - Port: &backupPort, - FullSnapshotSchedule: &snapshotSchedule, - GarbageCollectionPolicy: &garbageCollectionPolicy, - GarbageCollectionPeriod: &garbageCollectionPeriod, - DeltaSnapshotPeriod: &deltaSnapshotPeriod, - DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, - EtcdSnapshotTimeout: &etcdSnapshotTimeout, - - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("2Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("23m"), - "memory": testutils.ParseQuantity("128Mi"), - }, - }, - Store: &druidv1alpha1.StoreSpec{ - SecretRef: &corev1.SecretReference{ - Name: "etcd-backup", - }, - Container: &container, - Provider: &localProvider, - Prefix: prefix, - }, - } -} - -func getBackupStore(name string, provider druidv1alpha1.StorageProvider) *druidv1alpha1.StoreSpec { - return &druidv1alpha1.StoreSpec{ - Container: &container, - Prefix: name, - Provider: &provider, - SecretRef: &corev1.SecretReference{ - Name: "etcd-backup", - }, - } -} - -func getBackupStoreForLocal(name string) *druidv1alpha1.StoreSpec { - provider := druidv1alpha1.StorageProvider("local") - return &druidv1alpha1.StoreSpec{ - Container: &container, - Prefix: name, - Provider: &provider, - } -} diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 45d3b5b30..4c63fdc1f 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -10,6 +10,10 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,6 +21,424 @@ import ( "k8s.io/apimachinery/pkg/types" ) +var ( + deltaSnapshotPeriod = metav1.Duration{ + Duration: 300 * time.Second, + } + garbageCollectionPeriod = metav1.Duration{ + Duration: 43200 * time.Second, + } + clientPort int32 = 2379 + serverPort int32 = 2380 + backupPort int32 = 8080 + imageEtcd = "eu.gcr.io/gardener-project/gardener/etcd-wrapper:v0.1.0" + imageBR = "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.25.0" + snapshotSchedule = "0 */24 * * *" + defragSchedule = "0 */24 * * *" + container = "default.bkp" + storageCapacity = resource.MustParse("5Gi") + storageClass = "default" + priorityClassName = "class_priority" + deltaSnapShotMemLimit = resource.MustParse("100Mi") + autoCompactionMode = druidv1alpha1.Periodic + autoCompactionRetention = "2m" + quota = resource.MustParse("8Gi") + localProvider = druidv1alpha1.StorageProvider("Local") + prefix = "/tmp" + volumeClaimTemplateName = "etcd-main" + garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) + metricsBasic = druidv1alpha1.Basic + etcdSnapshotTimeout = metav1.Duration{ + Duration: 10 * time.Minute, + } + etcdDefragTimeout = metav1.Duration{ + Duration: 10 * time.Minute, + } +) + +type EtcdBuilder struct { + etcd *druidv1alpha1.Etcd +} + +func EtcdBuilderWithDefaults(name, namespace string) *EtcdBuilder { + builder := EtcdBuilder{} + builder.etcd = getDefaultEtcd(name, namespace) + return &builder +} + +func EtcdBuilderWithoutDefaults(name, namespace string) *EtcdBuilder { + builder := EtcdBuilder{} + builder.etcd = getEtcdWithoutDefaults(name, namespace) + return &builder +} + +func (eb *EtcdBuilder) WithReplicas(replicas int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Replicas = replicas + return eb +} + +func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + clientTLSConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, + DataKey: pointer.String("ca.crt"), + }, + ClientTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-client-tls", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-server-tls", + }, + } + + peerTLSConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "peer-url-ca-etcd", + }, + DataKey: pointer.String("ca.crt"), + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", + }, + } + + eb.etcd.Spec.Etcd.ClientUrlTLS = clientTLSConfig + eb.etcd.Spec.Etcd.PeerUrlTLS = peerTLSConfig + eb.etcd.Spec.Backup.TLS = clientTLSConfig + + return eb +} + +func (eb *EtcdBuilder) WithReadyStatus() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + members := make([]druidv1alpha1.EtcdMemberStatus, 0) + for i := 0; i < int(eb.etcd.Spec.Replicas); i++ { + members = append(members, druidv1alpha1.EtcdMemberStatus{Status: druidv1alpha1.EtcdMemberStatusReady}) + } + eb.etcd.Status = druidv1alpha1.EtcdStatus{ + ReadyReplicas: eb.etcd.Spec.Replicas, + Replicas: eb.etcd.Spec.Replicas, + CurrentReplicas: eb.etcd.Spec.Replicas, + UpdatedReplicas: eb.etcd.Spec.Replicas, + Ready: pointer.Bool(true), + Members: members, + Conditions: []druidv1alpha1.Condition{ + {Type: druidv1alpha1.ConditionTypeAllMembersReady, Status: druidv1alpha1.ConditionTrue}, + }, + } + + return eb +} + +func (eb *EtcdBuilder) WithStorageProvider(provider druidv1alpha1.StorageProvider) *EtcdBuilder { + // TODO: there is no default case right now which is not very right, returning an error in a default case makes it difficult to chain + // This should be improved later + switch provider { + case "aws": + return eb.WithProviderS3() + case "azure": + return eb.WithProviderABS() + case "alicloud": + return eb.WithProviderOSS() + case "gcp": + return eb.WithProviderGCS() + case "openstack": + return eb.WithProviderSwift() + case "local": + return eb.WithProviderLocal() + default: + return eb + } +} + +func (eb *EtcdBuilder) WithProviderS3() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "aws", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderABS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "azure", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderGCS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "gcp", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderSwift() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "openstack", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderOSS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStore( + eb.etcd.Name, + "alicloud", + ) + return eb +} + +func (eb *EtcdBuilder) WithProviderLocal() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Store = getBackupStoreForLocal( + eb.etcd.Name, + ) + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientPort(clientPort *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Etcd.ClientPort = clientPort + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + if eb.etcd.Spec.Etcd.ClientService == nil { + eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} + } + + eb.etcd.Spec.Etcd.ClientService.Labels = MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) + return eb +} + +func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]string) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + + if eb.etcd.Spec.Etcd.ClientService == nil { + eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} + } + + eb.etcd.Spec.Etcd.ClientService.Annotations = MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) + return eb +} + +func (eb *EtcdBuilder) WithEtcdServerPort(serverPort *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Etcd.ServerPort = serverPort + return eb +} + +func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } + eb.etcd.Spec.Backup.Port = port + return eb +} + +// WithLabels merges the labels that already exists with the ones that are passed to this method. If an entry is +// already present in the existing labels then it gets overwritten with the new value present in the passed in labels. +func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { + MergeMaps[string, string](eb.etcd.Labels, labels) + return eb +} + +// WithDefaultBackup creates a default backup spec and initializes etcd with it. +func (eb *EtcdBuilder) WithDefaultBackup() *EtcdBuilder { + eb.etcd.Spec.Backup = getBackupSpec() + return eb +} + +func (eb *EtcdBuilder) Build() *druidv1alpha1.Etcd { + return eb.etcd +} + +func getEtcdWithoutDefaults(name, namespace string) *druidv1alpha1.Etcd { + return &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: druidv1alpha1.EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + "role": "test", + }, + Labels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + }, + Replicas: 1, + Backup: druidv1alpha1.BackupSpec{}, + Etcd: druidv1alpha1.EtcdConfig{}, + Common: druidv1alpha1.SharedConfig{}, + }, + } +} + +func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { + return &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID(uuid.NewString()), + }, + Spec: druidv1alpha1.EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + "role": "test", + }, + Labels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd-statefulset", + "instance": name, + "name": "etcd", + }, + }, + Replicas: 1, + StorageCapacity: &storageCapacity, + StorageClass: &storageClass, + PriorityClassName: &priorityClassName, + VolumeClaimTemplate: &volumeClaimTemplateName, + Backup: getBackupSpec(), + Etcd: druidv1alpha1.EtcdConfig{ + Quota: "a, + Metrics: &metricsBasic, + Image: &imageEtcd, + DefragmentationSchedule: &defragSchedule, + EtcdDefragTimeout: &etcdDefragTimeout, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": ParseQuantity("2500m"), + "memory": ParseQuantity("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": ParseQuantity("500m"), + "memory": ParseQuantity("1000Mi"), + }, + }, + ClientPort: &clientPort, + ServerPort: &serverPort, + }, + Common: druidv1alpha1.SharedConfig{ + AutoCompactionMode: &autoCompactionMode, + AutoCompactionRetention: &autoCompactionRetention, + }, + }, + } +} + +func getBackupSpec() druidv1alpha1.BackupSpec { + return druidv1alpha1.BackupSpec{ + Image: &imageBR, + Port: &backupPort, + FullSnapshotSchedule: &snapshotSchedule, + GarbageCollectionPolicy: &garbageCollectionPolicy, + GarbageCollectionPeriod: &garbageCollectionPeriod, + DeltaSnapshotPeriod: &deltaSnapshotPeriod, + DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, + EtcdSnapshotTimeout: &etcdSnapshotTimeout, + + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": ParseQuantity("500m"), + "memory": ParseQuantity("2Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": ParseQuantity("23m"), + "memory": ParseQuantity("128Mi"), + }, + }, + Store: &druidv1alpha1.StoreSpec{ + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + Container: &container, + Provider: &localProvider, + Prefix: prefix, + }, + } +} + +func getBackupStore(name string, provider druidv1alpha1.StorageProvider) *druidv1alpha1.StoreSpec { + return &druidv1alpha1.StoreSpec{ + Container: &container, + Prefix: name, + Provider: &provider, + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + } +} + +func getBackupStoreForLocal(name string) *druidv1alpha1.StoreSpec { + provider := druidv1alpha1.StorageProvider("local") + return &druidv1alpha1.StoreSpec{ + Container: &container, + Prefix: name, + Provider: &provider, + } +} + // CheckEtcdOwnerReference checks if one of the owner references points to etcd owner UID. func CheckEtcdOwnerReference(refs []metav1.OwnerReference, etcdOwnerUID types.UID) bool { for _, ownerRef := range refs { From a66fbfa9a0d5359a9140cebeb1c03d9a4659e9c7 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 18 Jan 2024 17:12:29 +0530 Subject: [PATCH 077/235] harmonized labels on resources managed by druid --- api/v1alpha1/types_common.go | 8 ---- api/v1alpha1/types_constant.go | 22 ++++++++++ api/v1alpha1/types_etcd.go | 4 +- internal/common/constants.go | 22 +++++++--- .../etcdcopybackupstask/reconciler.go | 7 ++-- .../etcdcopybackupstask/reconciler_test.go | 6 +-- internal/health/etcdmember/check_ready.go | 9 ++-- .../operator/clientservice/clientservice.go | 16 +++++--- internal/operator/configmap/configmap.go | 41 ++++++++++++------- internal/operator/memberlease/memberlease.go | 28 ++++++++----- .../operator/memberlease/memberlease_test.go | 31 +++++--------- internal/operator/peerservice/peerservice.go | 11 ++++- .../poddisruptionbudget.go | 22 +++++----- .../poddisruptionbudget_test.go | 8 ---- internal/operator/role/role.go | 13 +++++- internal/operator/rolebinding/rolebinding.go | 13 +++++- .../operator/serviceaccount/serviceaccount.go | 13 +++++- .../operator/snapshotlease/snapshotlease.go | 28 ++++++++----- .../snapshotlease/snapshotlease_test.go | 30 +++++--------- internal/utils/envvar.go | 2 +- internal/utils/image.go | 31 +++++++------- test/utils/lease.go | 10 ++--- test/utils/stsmatcher.go | 1 + 23 files changed, 225 insertions(+), 151 deletions(-) create mode 100644 api/v1alpha1/types_constant.go create mode 100644 test/utils/stsmatcher.go diff --git a/api/v1alpha1/types_common.go b/api/v1alpha1/types_common.go index 5b601eeb7..301bc1090 100644 --- a/api/v1alpha1/types_common.go +++ b/api/v1alpha1/types_common.go @@ -63,11 +63,3 @@ type Condition struct { // A human readable message indicating details about the transition. Message string `json:"message"` } - -const ( - // IgnoreReconciliationAnnotation is an annotation set by an operator in order to stop reconciliation. - // Deprecated: Please use SuspendEtcdSpecReconcileAnnotation instead - IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" - // SuspendEtcdSpecReconcileAnnotation is an annotation set by an operator to temporarily suspend any etcd spec reconciliation. - SuspendEtcdSpecReconcileAnnotation = "druid.gardener.cloud/suspend-etcd-spec-reconcile" -) diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/types_constant.go new file mode 100644 index 000000000..88aed83f9 --- /dev/null +++ b/api/v1alpha1/types_constant.go @@ -0,0 +1,22 @@ +package v1alpha1 + +// Common label keys to be placed on all druid managed resources +const ( + // LabelAppNameKey is a label which sets the name of the resource provisioned for an etcd cluster. + LabelAppNameKey = "app.kubernetes.io/name" + // LabelManagedByKey is a label which sets druid as a manager for resources provisioned for an etcd cluster. + LabelManagedByKey = "app.kubernetes.io/managed-by" + // LabelPartOfKey is a label which establishes that a provisioned resource belongs to a parent etcd cluster. + LabelPartOfKey = "app.kubernetes.io/part-of" + // LabelComponentKey sets the component type label on resources provisioned for an etcd cluster. + LabelComponentKey = "app.kubernetes.io/component" +) + +// Annotation keys that can be placed on an etcd custom resource. +const ( + // IgnoreReconciliationAnnotation is an annotation set by an operator in order to stop reconciliation. + // Deprecated: Please use SuspendEtcdSpecReconcileAnnotation instead + IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" + // SuspendEtcdSpecReconcileAnnotation is an annotation set by an operator to temporarily suspend any etcd spec reconciliation. + SuspendEtcdSpecReconcileAnnotation = "druid.gardener.cloud/suspend-etcd-spec-reconcile" +) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index d9fefa478..72d1a9380 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -540,8 +540,8 @@ func (e *Etcd) GetMemberLeaseNames() []string { // GetDefaultLabels returns the default labels for etcd. func (e *Etcd) GetDefaultLabels() map[string]string { return map[string]string{ - "name": "etcd", - "instance": e.Name, + LabelManagedByKey: "etcd-druid", + LabelPartOfKey: e.Name, } } diff --git a/internal/common/constants.go b/internal/common/constants.go index 8a2571631..04bac2e1f 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -14,6 +14,7 @@ package common +// Constants for image keys const ( // Etcd is the key for the etcd image in the image vector. Etcd = "etcd" @@ -25,15 +26,27 @@ const ( BackupRestoreDistroless = "etcd-backup-restore-distroless" // Alpine is the key for the alpine image in the image vector. Alpine = "alpine" - // ChartPath is the directory containing the default image vector file. - ChartPath = "charts" - // GardenerOwnedBy is a constant for an annotation on a resource that describes the owner resource. +) + +// Constants defining keys used in resource labels and annotation keys. +const ( + // GardenerOwnedBy is a constant for an annotation/label on a resource that describes the owner resource. + // Deprecated: This key will be removed in later releases and code should not depend upon it. GardenerOwnedBy = "gardener.cloud/owned-by" - // GardenerOwnerType is a constant for an annotation on a resource that describes the type of owner resource. + // GardenerOwnerType is a constant for an annotation/label on a resource that describes the type of owner resource. + // Deprecated: This key will be removed in later releases and code should not depend upon it. GardenerOwnerType = "gardener.cloud/owner-type" +) + +const ( + // ChartPath is the directory containing the default image vector file. + ChartPath = "charts" // FinalizerName is the name of the etcd finalizer. FinalizerName = "druid.gardener.cloud/etcd-druid" +) +// Constants for environment variables +const ( // EnvPodName is the environment variable key for the pod name. EnvPodName = "POD_NAME" // EnvPodNamespace is the environment variable key for the pod namespace. @@ -42,7 +55,6 @@ const ( EnvStorageContainer = "STORAGE_CONTAINER" // EnvSourceStorageContainer is the environment variable key for the source storage container. EnvSourceStorageContainer = "SOURCE_STORAGE_CONTAINER" - // EnvAWSApplicationCredentials is the environment variable key for AWS application credentials. EnvAWSApplicationCredentials = "AWS_APPLICATION_CREDENTIALS" // EnvAzureApplicationCredentials is the environment variable key for Azure application credentials. diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 9c26ee86a..0439cfdb9 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -21,10 +21,11 @@ import ( "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" - ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/features" + ctrlutils "github.com/gardener/etcd-druid/controllers/utils" + "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/pkg/features" "github.com/gardener/etcd-druid/pkg/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index aa500b798..20d5bed28 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -20,9 +20,9 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/client/kubernetes" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index 996184a80..0cddf9c98 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -20,10 +20,6 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/utils" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" @@ -50,7 +46,10 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul leases := &coordinationv1.LeaseList{} if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ - common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: utils.PurposeMemberLease}); err != nil { + druidv1alpha1.LabelComponentKey: "member-lease", + druidv1alpha1.LabelManagedByKey: "etcd-druid", + druidv1alpha1.LabelPartOfKey: etcd.Name, + }); err != nil { r.logger.Error(err, "failed to get leases for etcd member readiness check") } diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 96ec436e6..70dcdae3c 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -28,6 +28,8 @@ const ( ErrSyncClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" ) +const componentName = "client-service" + type _resource struct { client client.Client } @@ -105,12 +107,16 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { } func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { - var labelMaps []map[string]string - labelMaps = append(labelMaps, etcd.GetDefaultLabels()) - if etcd.Spec.Etcd.ClientService != nil { - labelMaps = append(labelMaps, etcd.Spec.Etcd.ClientService.Labels) + clientSvcLabels := map[string]string{ + druidv1alpha1.LabelAppNameKey: etcd.GetClientServiceName(), + druidv1alpha1.LabelComponentKey: componentName, + } + // Add any client service labels as defined in the etcd resource + specClientSvcLabels := map[string]string{} + if etcd.Spec.Etcd.ClientService != nil && etcd.Spec.Etcd.ClientService.Labels != nil { + specClientSvcLabels = etcd.Spec.Etcd.ClientService.Labels } - return utils.MergeMaps[string, string](labelMaps...) + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), clientSvcLabels, specClientSvcLabels) } func getAnnotations(etcd *druidv1alpha1.Etcd) map[string]string { diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 776244d5c..85b50a685 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -5,8 +5,9 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils" + gardenerutils "github.com/gardener/gardener/pkg/utils" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -32,19 +33,9 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - cfg := createEtcdConfig(etcd) - cfgYaml, err := yaml.Marshal(cfg) - if err != nil { - return err - } cm := emptyConfigMap(getObjectKey(etcd)) - _, err = controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { - cm.Name = etcd.GetConfigMapName() - cm.Namespace = etcd.Namespace - cm.Labels = etcd.GetDefaultLabels() - cm.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - cm.Data = map[string]string{"etcd.conf.yaml": string(cfgYaml)} - return nil + _, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { + return buildResource(etcd, cm) }) if err != nil { return err @@ -69,6 +60,28 @@ func New(client client.Client) resource.Operator { } } +func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { + cfg := createEtcdConfig(etcd) + cfgYaml, err := yaml.Marshal(cfg) + if err != nil { + return err + } + cm.Name = etcd.GetConfigMapName() + cm.Namespace = etcd.Namespace + cm.Labels = getLabels(etcd) + cm.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + cm.Data = map[string]string{"etcd.conf.yaml": string(cfgYaml)} + return nil +} + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + cmLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: "etcd-config", + druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), cmLabels) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{ Name: etcd.GetConfigMapName(), @@ -90,5 +103,5 @@ func computeCheckSum(cm *corev1.ConfigMap) (string, error) { if err != nil { return "", nil } - return utils.ComputeSHA256Hex(jsonData), nil + return gardenerutils.ComputeSHA256Hex(jsonData), nil } diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index eaf69b341..1b3427788 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -4,13 +4,11 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/hashicorp/go-multierror" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,7 +21,7 @@ const ( ErrSyncMemberLease druidv1alpha1.ErrorCode = "ERR_SYNC_MEMBER_LEASE" ) -const purpose = "etcd-member-lease" +const componentName = "member-lease" type _resource struct { client client.Client @@ -35,7 +33,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * err := r.client.List(ctx, leaseList, client.InNamespace(etcd.Namespace), - client.MatchingLabels(getLabels(etcd))) + client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd))) if err != nil { return resourceNames, druiderr.WrapError(err, ErrListMemberLease, @@ -91,7 +89,7 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(etcd.Namespace), - client.MatchingLabels(getLabels(etcd))); err != nil { + client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd))); err != nil { return druiderr.WrapError(err, ErrDeleteMemberLease, "TriggerDelete", @@ -108,7 +106,7 @@ func New(client client.Client) resource.Operator { } func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { - lease.Labels = utils.MergeMaps[string](utils.GetMemberLeaseLabels(etcd.Name), etcd.GetDefaultLabels()) + lease.Labels = getLabels(etcd, lease.Name) lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} } @@ -121,11 +119,19 @@ func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { return objectKeys } -func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { - labels := make(map[string]string) - labels[common.GardenerOwnedBy] = etcd.Name - labels[v1beta1constants.GardenerPurpose] = purpose - return utils.MergeMaps[string, string](labels, etcd.GetDefaultLabels()) +func getSelectorLabelsForAllMemberLeases(etcd *druidv1alpha1.Etcd) map[string]string { + leaseMatchingLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), leaseMatchingLabels) +} + +func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { + leaseLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: leaseName, + } + return utils.MergeMaps[string, string](leaseLabels, etcd.GetDefaultLabels()) } func emptyMemberLease(objectKey client.ObjectKey) *coordinationv1.Lease { diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 2e54211f8..a4a4100b1 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -7,11 +7,10 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" @@ -254,7 +253,7 @@ func TestTriggerDelete(t *testing.T) { } } for _, nonTargetLeaseName := range nonTargetLeaseNames { - fakeClientBuilder.WithObjects(testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, purpose)) + fakeClientBuilder.WithObjects(testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, componentName)) } cl := fakeClientBuilder.Build() // ***************** Setup operator and test ***************** @@ -269,7 +268,7 @@ func TestTriggerDelete(t *testing.T) { g.Expect(memberLeasesPostDelete).Should(HaveLen(tc.numExistingLeases)) } else { g.Expect(memberLeasesPostDelete).Should(HaveLen(0)) - nonTargetMemberLeases := getLatestNonTargetMemberLeases(g, cl, nonTargetEtcd) + nonTargetMemberLeases := getLatestMemberLeases(g, cl, nonTargetEtcd) g.Expect(nonTargetMemberLeases).To(HaveLen(len(nonTargetLeaseNames))) g.Expect(nonTargetMemberLeases).To(ConsistOf(memberLeases(nonTargetEtcd.Name, nonTargetEtcd.UID, int32(len(nonTargetLeaseNames))))) } @@ -279,14 +278,12 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { - return doGetLatestLeases(g, cl, etcd, map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: purpose, - }) -} - -func getLatestNonTargetMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) []coordinationv1.Lease { - return doGetLatestLeases(g, cl, etcd, getAdditionalMemberLeaseLabels(etcd.Name)) + return doGetLatestLeases(g, + cl, + etcd, + utils.MergeMaps[string, string](map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + }, etcd.GetDefaultLabels())) } func doGetLatestLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) []coordinationv1.Lease { @@ -320,7 +317,6 @@ func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 if numLeases > int(etcd.Spec.Replicas) { return nil, errors.New("number of requested leases is greater than the etcd replicas") } - additionalLabels := getAdditionalMemberLeaseLabels(etcd.Name) memberLeaseNames := etcd.GetMemberLeaseNames() leases := make([]*coordinationv1.Lease, 0, numLeases) for i := 0; i < numLeases; i++ { @@ -328,7 +324,7 @@ func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 ObjectMeta: metav1.ObjectMeta{ Name: memberLeaseNames[i], Namespace: etcd.Namespace, - Labels: testutils.MergeMaps[string, string](etcd.GetDefaultLabels(), additionalLabels), + Labels: getLabels(etcd, memberLeaseNames[i]), OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, }, } @@ -336,10 +332,3 @@ func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 } return leases, nil } - -func getAdditionalMemberLeaseLabels(etcdName string) map[string]string { - return map[string]string{ - common.GardenerOwnedBy: etcdName, - v1beta1constants.GardenerPurpose: purpose, - } -} diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 278a0b334..5ba225fc7 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -16,6 +16,7 @@ import ( ) const defaultServerPort = 2380 +const componentName = "peer-service" const ( ErrGetPeerService druidv1alpha1.ErrorCode = "ERR_GET_PEER_SERVICE" @@ -84,7 +85,7 @@ func New(client client.Client) resource.Operator { } func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { - svc.Labels = etcd.GetDefaultLabels() + svc.Labels = getLabels(etcd) svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} svc.Spec.Type = corev1.ServiceTypeClusterIP svc.Spec.ClusterIP = corev1.ClusterIPNone @@ -94,6 +95,14 @@ func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { svc.Spec.Ports = getPorts(etcd) } +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + svcLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: etcd.GetPeerServiceName(), + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), svcLabels) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace} } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 6f0949374..562d8e0cb 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -4,9 +4,9 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -21,6 +21,8 @@ const ( ErrSyncPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_SYNC_POD_DISRUPTION_BUDGET" ) +const componentName = "etcd-pdb" + type _resource struct { client client.Client } @@ -77,8 +79,7 @@ func New(client client.Client) resource.Operator { } func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) { - pdb.Labels = etcd.GetDefaultLabels() - pdb.Annotations = getAnnotations(etcd) + pdb.Labels = getLabels(etcd) pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} pdb.Spec.MinAvailable = &intstr.IntOrString{ IntVal: computePDBMinAvailable(int(etcd.Spec.Replicas)), @@ -89,6 +90,14 @@ func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) } } +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + pdbLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: etcd.Name, + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), pdbLabels) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace} } @@ -102,13 +111,6 @@ func emptyPodDisruptionBudget(objectKey client.ObjectKey) *policyv1.PodDisruptio } } -func getAnnotations(etcd *druidv1alpha1.Etcd) map[string]string { - return map[string]string{ - common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - common.GardenerOwnerType: "etcd", - } -} - func computePDBMinAvailable(etcdReplicas int) int32 { // do not enable PDB for single node cluster if etcdReplicas <= 1 { diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 0661f43d0..de19caa68 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -3,11 +3,9 @@ package poddistruptionbudget import ( "context" "errors" - "fmt" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" testutils "github.com/gardener/etcd-druid/test/utils" @@ -265,17 +263,11 @@ func getLatestPodDisruptionBudget(cl client.Client, etcd *druidv1alpha1.Etcd) (* } func matchPodDisruptionBudget(g *WithT, etcd *druidv1alpha1.Etcd, actualPDB policyv1.PodDisruptionBudget, expectedPDBMinAvailable int32) { - expectedAnnotations := map[string]string{ - common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - common.GardenerOwnerType: "etcd", - } - g.Expect(actualPDB).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(etcd.Name), "Namespace": Equal(etcd.Namespace), "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), - "Annotations": testutils.MatchResourceAnnotations(expectedAnnotations), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), "Spec": MatchFields(IgnoreExtras, Fields{ diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 205c1ad5c..ca37025a7 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -6,6 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -19,6 +20,8 @@ const ( ErrDeleteRole druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE" ) +const componentName = "druid-role" + type _resource struct { client client.Client } @@ -89,7 +92,7 @@ func emptyRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { } func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { - role.Labels = etcd.GetDefaultLabels() + role.Labels = getLabels(etcd) role.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} role.Rules = []rbacv1.PolicyRule{ { @@ -109,3 +112,11 @@ func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { }, } } + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + roleLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: etcd.GetRoleName(), + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) +} diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index eec1c1496..c46d6bd27 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -6,6 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -19,6 +20,8 @@ const ( ErrDeleteRoleBinding druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE_BINDING" ) +const componentName = "druid-role-binding" + type _resource struct { client client.Client } @@ -100,7 +103,7 @@ func emptyRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { } func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { - rb.Labels = etcd.GetDefaultLabels() + rb.Labels = getLabels(etcd) rb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} rb.RoleRef = rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -115,3 +118,11 @@ func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { }, } } + +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + roleLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: etcd.GetRoleBindingName(), + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) +} diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 7f7eaa54b..9d45891a8 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -6,6 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -20,6 +21,8 @@ const ( ErrSyncServiceAccount druidv1alpha1.ErrorCode = "ERR_SYNC_SERVICE_ACCOUNT" ) +const componentName = "druid-service-account" + type _resource struct { client client.Client disableAutoMount bool @@ -79,11 +82,19 @@ func New(client client.Client, disableAutomount bool) resource.Operator { } func buildResource(etcd *druidv1alpha1.Etcd, sa *corev1.ServiceAccount, autoMountServiceAccountToken bool) { - sa.Labels = etcd.GetDefaultLabels() + sa.Labels = getLabels(etcd) sa.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} sa.AutomountServiceAccountToken = pointer.Bool(autoMountServiceAccountToken) } +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + roleLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: etcd.GetServiceAccountName(), + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) +} + func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace} } diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index b6b93ecd1..62e070afe 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -5,11 +5,9 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" "github.com/hashicorp/go-multierror" coordinationv1 "k8s.io/api/coordination/v1" @@ -18,14 +16,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const purpose = "etcd-snapshot-lease" - const ( ErrGetSnapshotLease druidv1alpha1.ErrorCode = "ERR_GET_SNAPSHOT_LEASE" ErrDeleteSnapshotLease druidv1alpha1.ErrorCode = "ERR_DELETE_SNAPSHOT_LEASE" ErrSyncSnapshotLease druidv1alpha1.ErrorCode = "ERR_SYNC_SNAPSHOT_LEASE" ) +const componentName = "snapshot-lease" + type _resource struct { client client.Client } @@ -112,7 +110,7 @@ func (r _resource) deleteAllSnapshotLeases(ctx resource.OperatorContext, etcd *d if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(etcd.Namespace), - client.MatchingLabels(getLabels(etcd))); err != nil { + client.MatchingLabels(getSelectorLabelsForAllSnapshotLeases(etcd))); err != nil { return wrapErrFn(err) } return nil @@ -153,15 +151,23 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a } func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { - lease.Labels = getLabels(etcd) + lease.Labels = getLabels(etcd, lease.Name) lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} } -func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { - labels := make(map[string]string) - labels[common.GardenerOwnedBy] = etcd.Name - labels[v1beta1constants.GardenerPurpose] = purpose - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), labels) +func getSelectorLabelsForAllSnapshotLeases(etcd *druidv1alpha1.Etcd) map[string]string { + leaseMatchingLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), leaseMatchingLabels) +} + +func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { + leaseLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: leaseName, + } + return utils.MergeMaps[string, string](leaseLabels, etcd.GetDefaultLabels()) } func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 5e49e9b49..797956b95 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -7,12 +7,10 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" @@ -184,7 +182,7 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { } else { g.Expect(latestSnapshotLeases).To(HaveLen(0)) // To ensure that delete of snapshot leases did not remove non-target- snapshot leases also check that these still exist - actualNonTargetSnapshotLeases, nonTargetSnapshotListErr := getLatestNonTargetSnapshotLeases(cl, nonTargetEtcd) + actualNonTargetSnapshotLeases, nonTargetSnapshotListErr := getLatestSnapshotLeases(cl, nonTargetEtcd) g.Expect(nonTargetSnapshotListErr).To(BeNil()) g.Expect(actualNonTargetSnapshotLeases).To(HaveLen(2)) } @@ -253,7 +251,7 @@ func TestTriggerDelete(t *testing.T) { g.Expect(snapshotLeaseListErr).ToNot(HaveOccurred()) g.Expect(latestSnapshotLeases).To(HaveLen(0)) } - actualNonTargetSnapshotLeases, nonTargetSnapshotListErr := getLatestNonTargetSnapshotLeases(cl, nonTargetEtcd) + actualNonTargetSnapshotLeases, nonTargetSnapshotListErr := getLatestSnapshotLeases(cl, nonTargetEtcd) g.Expect(nonTargetSnapshotListErr).ToNot(HaveOccurred()) g.Expect(actualNonTargetSnapshotLeases).To(HaveLen(2)) }) @@ -277,8 +275,8 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas Name: leaseName, Namespace: etcd.Namespace, Labels: utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ - "gardener.cloud/owned-by": etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: leaseName, }), OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, }, @@ -287,8 +285,8 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: purpose, + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: leaseName, }) return MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ @@ -300,18 +298,12 @@ func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMa }) } -func getLatestNonTargetSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { - return doGetLatestLeases(cl, etcd, map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-snapshot-lease", - }) -} - func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { - return doGetLatestLeases(cl, etcd, map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: purpose, - }) + return doGetLatestLeases(cl, + etcd, + utils.MergeMaps[string, string](map[string]string{ + druidv1alpha1.LabelComponentKey: componentName, + }, etcd.GetDefaultLabels())) } func doGetLatestLeases(cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) ([]coordinationv1.Lease, error) { diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index a3d997635..5342b030e 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -4,7 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" diff --git a/internal/utils/image.go b/internal/utils/image.go index 1d9996e59..01e604be5 100755 --- a/internal/utils/image.go +++ b/internal/utils/image.go @@ -16,36 +16,23 @@ package utils import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/gardener/pkg/utils/imagevector" "k8s.io/utils/pointer" ) -func getEtcdImageKeys(useEtcdWrapper bool) (etcdImageKey string, etcdbrImageKey string, alpine string) { - alpine = common.Alpine - switch useEtcdWrapper { - case true: - etcdImageKey = common.EtcdWrapper - etcdbrImageKey = common.BackupRestoreDistroless - default: - etcdImageKey = common.Etcd - etcdbrImageKey = common.BackupRestore - } - return -} - // GetEtcdImages returns images for etcd and backup-restore by inspecting the etcd spec and the image vector // and returns the image for the init container by inspecting the image vector. // It will give preference to images that are set in the etcd spec and only if the image is not found in it should // it be picked up from the image vector if it's set there. // A return value of nil for either of the images indicates that the image is not set. func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcdWrapper bool) (string, string, string, error) { - etcdImageKey, etcdbrImageKey, initContainerImageKey := getEtcdImageKeys(useEtcdWrapper) + etcdImageKey, etcdBRImageKey, initContainerImageKey := getEtcdImageKeys(useEtcdWrapper) etcdImage, err := chooseImage(etcdImageKey, etcd.Spec.Etcd.Image, iv) if err != nil { return "", "", "", err } - etcdBackupRestoreImage, err := chooseImage(etcdbrImageKey, etcd.Spec.Backup.Image, iv) + etcdBackupRestoreImage, err := chooseImage(etcdBRImageKey, etcd.Spec.Backup.Image, iv) if err != nil { return "", "", "", err } @@ -57,6 +44,18 @@ func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcd return *etcdImage, *etcdBackupRestoreImage, *initContainerImage, nil } +func getEtcdImageKeys(useEtcdWrapper bool) (etcdImageKey string, etcdBRImageKey string, alpine string) { + alpine = common.Alpine + if useEtcdWrapper { + etcdImageKey = common.EtcdWrapper + etcdBRImageKey = common.BackupRestoreDistroless + } else { + etcdImageKey = common.Etcd + etcdBRImageKey = common.BackupRestore + } + return +} + // chooseImage selects an image based on the given key, specImage, and image vector. // It returns the specImage if it is not nil; otherwise, it searches for the image in the image vector. func chooseImage(key string, specImage *string, iv imagevector.ImageVector) (*string, error) { diff --git a/test/utils/lease.go b/test/utils/lease.go index 335bc00fc..1cb4c561c 100644 --- a/test/utils/lease.go +++ b/test/utils/lease.go @@ -19,16 +19,16 @@ import ( ) // CreateLease creates a lease with its owner reference set to etcd. -func CreateLease(name, namespace, etcdName string, etcdUID types.UID, purpose string) *coordinationv1.Lease { +func CreateLease(name, namespace, etcdName string, etcdUID types.UID, componentName string) *coordinationv1.Lease { return &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: map[string]string{ - "name": "etcd", - "instance": etcdName, - "gardener.cloud/owned-by": etcdName, - "gardener.cloud/purpose": purpose, + druidv1alpha1.LabelPartOfKey: etcdName, + druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelAppNameKey: name, + druidv1alpha1.LabelManagedByKey: "etcd-druid", }, OwnerReferences: []metav1.OwnerReference{{ APIVersion: druidv1alpha1.GroupVersion.String(), diff --git a/test/utils/stsmatcher.go b/test/utils/stsmatcher.go new file mode 100644 index 000000000..d4b585bf7 --- /dev/null +++ b/test/utils/stsmatcher.go @@ -0,0 +1 @@ +package utils From ff10a01e05ead52c59be73e053d1c362798b69ae Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 19 Jan 2024 12:34:32 +0530 Subject: [PATCH 078/235] moved the component names to constants and adpated code --- api/v1alpha1/types_constant.go | 8 ++++--- api/v1alpha1/types_etcd.go | 2 +- internal/common/constants.go | 24 +++++++++++-------- internal/health/etcdmember/check_ready.go | 5 ++-- .../health/etcdmember/check_ready_test.go | 9 +++---- .../operator/clientservice/clientservice.go | 5 ++-- internal/operator/configmap/configmap.go | 3 ++- internal/operator/memberlease/memberlease.go | 7 +++--- internal/operator/peerservice/peerservice.go | 4 ++-- .../poddisruptionbudget.go | 5 ++-- internal/operator/role/role.go | 5 ++-- internal/operator/rolebinding/rolebinding.go | 5 ++-- .../operator/serviceaccount/serviceaccount.go | 5 ++-- .../operator/snapshotlease/snapshotlease.go | 7 +++--- 14 files changed, 48 insertions(+), 46 deletions(-) diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/types_constant.go index 88aed83f9..3a14adfa2 100644 --- a/api/v1alpha1/types_constant.go +++ b/api/v1alpha1/types_constant.go @@ -4,11 +4,13 @@ package v1alpha1 const ( // LabelAppNameKey is a label which sets the name of the resource provisioned for an etcd cluster. LabelAppNameKey = "app.kubernetes.io/name" - // LabelManagedByKey is a label which sets druid as a manager for resources provisioned for an etcd cluster. + // LabelManagedByKey is a key of a label which sets druid as a manager for resources provisioned for an etcd cluster. LabelManagedByKey = "app.kubernetes.io/managed-by" - // LabelPartOfKey is a label which establishes that a provisioned resource belongs to a parent etcd cluster. + // LabelManagedByValue is the value for LabelManagedByKey. + LabelManagedByValue = "etcd-druid" + // LabelPartOfKey is a key of a label which establishes that a provisioned resource belongs to a parent etcd cluster. LabelPartOfKey = "app.kubernetes.io/part-of" - // LabelComponentKey sets the component type label on resources provisioned for an etcd cluster. + // LabelComponentKey is a key for a label that sets the component type on resources provisioned for an etcd cluster. LabelComponentKey = "app.kubernetes.io/component" ) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 72d1a9380..0818835ad 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -540,7 +540,7 @@ func (e *Etcd) GetMemberLeaseNames() []string { // GetDefaultLabels returns the default labels for etcd. func (e *Etcd) GetDefaultLabels() map[string]string { return map[string]string{ - LabelManagedByKey: "etcd-druid", + LabelManagedByKey: LabelManagedByValue, LabelPartOfKey: e.Name, } } diff --git a/internal/common/constants.go b/internal/common/constants.go index 04bac2e1f..bdfc77d19 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -28,16 +28,6 @@ const ( Alpine = "alpine" ) -// Constants defining keys used in resource labels and annotation keys. -const ( - // GardenerOwnedBy is a constant for an annotation/label on a resource that describes the owner resource. - // Deprecated: This key will be removed in later releases and code should not depend upon it. - GardenerOwnedBy = "gardener.cloud/owned-by" - // GardenerOwnerType is a constant for an annotation/label on a resource that describes the type of owner resource. - // Deprecated: This key will be removed in later releases and code should not depend upon it. - GardenerOwnerType = "gardener.cloud/owner-type" -) - const ( // ChartPath is the directory containing the default image vector file. ChartPath = "charts" @@ -76,3 +66,17 @@ const ( // EnvECSSecretAccessKey is the environment variable key for Dell ECS secret access key. EnvECSSecretAccessKey = "ECS_SECRET_ACCESS_KEY" ) + +// Constants for values to be set against druidv1alpha1.LabelComponentKey +const ( + ClientServiceComponentName = "etcd-client-service" + ConfigMapComponentName = "etcd-config" + MemberLeaseComponentName = "etcd-member-lease" + SnapshotLeaseComponentName = "etcd-snapshot-lease" + PeerServiceComponentName = "etcd-peer-service" + PodDisruptionBudgetComponentName = "etcd-pdb" + RoleComponentName = "etcd-druid-role" + RoleBindingComponentName = "druid-role-binding" + ServiceAccountComponentName = "druid-service-account" + StatefulSetComponentName = "etcd-sts" +) diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index 0cddf9c98..c0c6069eb 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -20,6 +20,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" @@ -46,8 +47,8 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul leases := &coordinationv1.LeaseList{} if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ - druidv1alpha1.LabelComponentKey: "member-lease", - druidv1alpha1.LabelManagedByKey: "etcd-druid", + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: etcd.Name, }); err != nil { r.logger.Error(err, "failed to get leases for etcd member readiness check") diff --git a/internal/health/etcdmember/check_ready_test.go b/internal/health/etcdmember/check_ready_test.go index f36eae29a..250c341d4 100644 --- a/internal/health/etcdmember/check_ready_test.go +++ b/internal/health/etcdmember/check_ready_test.go @@ -24,9 +24,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" . "github.com/gardener/etcd-druid/internal/health/etcdmember" mockclient "github.com/gardener/etcd-druid/internal/mock/controller-runtime/client" - "github.com/gardener/etcd-druid/internal/utils" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/test" "github.com/go-logr/logr" @@ -88,7 +85,11 @@ var _ = Describe("ReadyCheck", func() { JustBeforeEach(func() { cl.EXPECT().List(ctx, gomock.AssignableToTypeOf(&coordinationv1.LeaseList{}), client.InNamespace(etcd.Namespace), - client.MatchingLabels{common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: utils.PurposeMemberLease}). + client.MatchingLabels{ + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: etcd.Name, + }). DoAndReturn( func(_ context.Context, leases *coordinationv1.LeaseList, _ ...client.ListOption) error { *leases = *leasesList diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 70dcdae3c..ab311bf2f 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -28,8 +29,6 @@ const ( ErrSyncClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" ) -const componentName = "client-service" - type _resource struct { client client.Client } @@ -109,7 +108,7 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { clientSvcLabels := map[string]string{ druidv1alpha1.LabelAppNameKey: etcd.GetClientServiceName(), - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.ClientServiceComponentName, } // Add any client service labels as defined in the etcd resource specClientSvcLabels := map[string]string{} diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 85b50a685..6f8f6a957 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -4,6 +4,7 @@ import ( "encoding/json" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -76,7 +77,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { cmLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: "etcd-config", + druidv1alpha1.LabelComponentKey: common.ConfigMapComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), cmLabels) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 1b3427788..8b8737ee3 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -21,8 +22,6 @@ const ( ErrSyncMemberLease druidv1alpha1.ErrorCode = "ERR_SYNC_MEMBER_LEASE" ) -const componentName = "member-lease" - type _resource struct { client client.Client } @@ -121,14 +120,14 @@ func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { func getSelectorLabelsForAllMemberLeases(etcd *druidv1alpha1.Etcd) map[string]string { leaseMatchingLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { leaseLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, } return utils.MergeMaps[string, string](leaseLabels, etcd.GetDefaultLabels()) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 5ba225fc7..6ed38c047 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -16,7 +17,6 @@ import ( ) const defaultServerPort = 2380 -const componentName = "peer-service" const ( ErrGetPeerService druidv1alpha1.ErrorCode = "ERR_GET_PEER_SERVICE" @@ -97,7 +97,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { svcLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.PeerServiceComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetPeerServiceName(), } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), svcLabels) diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 562d8e0cb..fc1c5e5eb 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -21,8 +22,6 @@ const ( ErrSyncPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_SYNC_POD_DISRUPTION_BUDGET" ) -const componentName = "etcd-pdb" - type _resource struct { client client.Client } @@ -92,7 +91,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { pdbLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.PodDisruptionBudgetComponentName, druidv1alpha1.LabelAppNameKey: etcd.Name, } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), pdbLabels) diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index ca37025a7..4718a8e17 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -20,8 +21,6 @@ const ( ErrDeleteRole druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE" ) -const componentName = "druid-role" - type _resource struct { client client.Client } @@ -115,7 +114,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.RoleComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetRoleName(), } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index c46d6bd27..e092f2daa 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -20,8 +21,6 @@ const ( ErrDeleteRoleBinding druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE_BINDING" ) -const componentName = "druid-role-binding" - type _resource struct { client client.Client } @@ -121,7 +120,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.RoleBindingComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetRoleBindingName(), } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 9d45891a8..01235b546 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -21,8 +22,6 @@ const ( ErrSyncServiceAccount druidv1alpha1.ErrorCode = "ERR_SYNC_SERVICE_ACCOUNT" ) -const componentName = "druid-service-account" - type _resource struct { client client.Client disableAutoMount bool @@ -89,7 +88,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, sa *corev1.ServiceAccount, autoMoun func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.ServiceAccountComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetServiceAccountName(), } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 62e070afe..2a262dfc5 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -5,6 +5,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -22,8 +23,6 @@ const ( ErrSyncSnapshotLease druidv1alpha1.ErrorCode = "ERR_SYNC_SNAPSHOT_LEASE" ) -const componentName = "snapshot-lease" - type _resource struct { client client.Client } @@ -157,14 +156,14 @@ func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { func getSelectorLabelsForAllSnapshotLeases(etcd *druidv1alpha1.Etcd) map[string]string { leaseMatchingLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { leaseLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, } return utils.MergeMaps[string, string](leaseLabels, etcd.GetDefaultLabels()) From e2bfea3ca14e1583a0a5785c330c45df68f9e3a5 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 19 Jan 2024 19:18:44 +0530 Subject: [PATCH 079/235] Add resource-protection webhook, aka "sentinel" webhook --- api/v1alpha1/types_constant.go | 2 + charts/druid/resources/ca.crt | 18 ++ charts/druid/resources/ca.csr | 15 ++ charts/druid/resources/ca.csr.conf | 4 + charts/druid/resources/ca.key | 27 ++ charts/druid/resources/ca.srl | 1 + charts/druid/resources/server.crt | 23 ++ charts/druid/resources/server.csr | 20 ++ charts/druid/resources/server.csr.conf | 28 ++ charts/druid/resources/server.key | 27 ++ .../templates/controller-deployment.yaml | 30 ++- charts/druid/templates/secret-ca-crt.yaml | 10 + .../templates/secret-server-tls-crt.yaml | 13 + charts/druid/templates/service.yaml | 21 ++ .../templates/validating-webhook-config.yaml | 91 +++++++ charts/druid/values.yaml | 9 + internal/controller/config.go | 70 ++++- internal/controller/etcd/reconcile_spec.go | 1 + internal/controller/etcd/reconciler.go | 1 + internal/controller/manager.go | 27 +- internal/webhook/sentinel/config.go | 21 ++ internal/webhook/sentinel/handler.go | 243 ++++++++++++++++++ internal/webhook/sentinel/register.go | 24 ++ 23 files changed, 719 insertions(+), 7 deletions(-) create mode 100644 charts/druid/resources/ca.crt create mode 100644 charts/druid/resources/ca.csr create mode 100644 charts/druid/resources/ca.csr.conf create mode 100644 charts/druid/resources/ca.key create mode 100644 charts/druid/resources/ca.srl create mode 100644 charts/druid/resources/server.crt create mode 100644 charts/druid/resources/server.csr create mode 100644 charts/druid/resources/server.csr.conf create mode 100644 charts/druid/resources/server.key create mode 100644 charts/druid/templates/secret-ca-crt.yaml create mode 100644 charts/druid/templates/secret-server-tls-crt.yaml create mode 100644 charts/druid/templates/service.yaml create mode 100644 charts/druid/templates/validating-webhook-config.yaml create mode 100644 internal/webhook/sentinel/config.go create mode 100644 internal/webhook/sentinel/handler.go create mode 100644 internal/webhook/sentinel/register.go diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/types_constant.go index 3a14adfa2..1491fdf1d 100644 --- a/api/v1alpha1/types_constant.go +++ b/api/v1alpha1/types_constant.go @@ -21,4 +21,6 @@ const ( IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" // SuspendEtcdSpecReconcileAnnotation is an annotation set by an operator to temporarily suspend any etcd spec reconciliation. SuspendEtcdSpecReconcileAnnotation = "druid.gardener.cloud/suspend-etcd-spec-reconcile" + // ResourceProtectionAnnotation is an annotation set by an operator to enable or disable protection of resources created by etcd-druid. + ResourceProtectionAnnotation = "druid.gardener.cloud/resource-protection" ) diff --git a/charts/druid/resources/ca.crt b/charts/druid/resources/ca.crt new file mode 100644 index 000000000..e7f6e70e4 --- /dev/null +++ b/charts/druid/resources/ca.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC7zCCAdegAwIBAgIJANukyPQUqbxMMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV +BAMMCmV0Y2QtZHJ1aWQwHhcNMjQwMTE4MTUzMTIzWhcNMzQwMTE4MTUzMTIzWjAV +MRMwEQYDVQQDDApldGNkLWRydWlkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAyluLT/y9ublUuRt40NQmqE5EIlejDNU4gDGwX2w+kzDvtlWlbQtI7PIZ +Y6R+Y/aqv8xb8+wGKeXC8lkRCPlf3HtjSULVRI3buAwoV82vb586T9gD78rJTwXW +qrmdsnUPdyS00JQPcgwO17AoCIyt2ErrT3ZZr0Zl1RlQfhM74UZMn/1y2j5AfQ2O +hvzgfbyzdvMOHrrwx4z1nPlV3OSs7Xj7NC9P6xnpSU552qunFfL7Tsu79485gYgp +8V+LohrV41gFZceVXzcy5LhF+OwxPFHm+2ReY6vRlf2U9KiSO061PNs/CbIaX6ns +f0/hMRXnmzsD6UMswFqOD8Ql7U53CQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAaYw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUqVPowEtxjAoOLm9YzeHHg0E2gVow +DQYJKoZIhvcNAQELBQADggEBADwdgnOnO9+V6cou48gW3yKN5XL6bdFnfnx9jWeM +dzPlskAkHJO0/MzjnMCRqqBdDj36c7/oyIpJAjYufak1rCqokTJG9SsNQ8VRfOAR +DQSJ4bk+33POoLJske3svoY3DOsxxG6MOJ9KoZE1ZylDwxkaZ4SmvoSfM+fPu7jZ +zp5EbWVq4bbTb8+FvjH49nf5xf5XrKm2ugwYeb0ZzgbB+HgMHto0kVqqNmYkl623 +t4PNWt5rOlck0wWJyUYdwN6MjAd+eVZBbQG47MjY52SzpplcPuRniO5CKJoBYve5 +fsMHHGGWxH4obB6OVg+H3JI4jQZwSQvFEPJrnK5utF7rJl4= +-----END CERTIFICATE----- diff --git a/charts/druid/resources/ca.csr b/charts/druid/resources/ca.csr new file mode 100644 index 000000000..312f71cbd --- /dev/null +++ b/charts/druid/resources/ca.csr @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICWjCCAUICAQAwFTETMBEGA1UEAwwKZXRjZC1kcnVpZDCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMpbi0/8vbm5VLkbeNDUJqhORCJXowzVOIAxsF9s +PpMw77ZVpW0LSOzyGWOkfmP2qr/MW/PsBinlwvJZEQj5X9x7Y0lC1USN27gMKFfN +r2+fOk/YA+/KyU8F1qq5nbJ1D3cktNCUD3IMDtewKAiMrdhK6092Wa9GZdUZUH4T +O+FGTJ/9cto+QH0Njob84H28s3bzDh668MeM9Zz5VdzkrO14+zQvT+sZ6UlOedqr +pxXy+07Lu/ePOYGIKfFfi6Ia1eNYBWXHlV83MuS4RfjsMTxR5vtkXmOr0ZX9lPSo +kjtOtTzbPwmyGl+p7H9P4TEV55s7A+lDLMBajg/EJe1OdwkCAwEAAaAAMA0GCSqG +SIb3DQEBCwUAA4IBAQB3eVZSclpxQV59UQVoEqCpnvlayBxleViV2bQR3R3rU/fW +iu/ZGWPLYuDmgt6PnZUoN2GlrYzFr72rXPKShJq7HsIaj3GuVkjmeXsr9I6MCTn+ +YzmmVZf2p1ZQhW0uUnrFkV/SLVv1r9abCCwNl4KU8STvUraHnoHhbw6wkJoKK5tr +dHnrFyk5fOwOQEyS1O6SZ9YVaYe3a6X10a/PyxmvO3GvAJYFXCjhqvTGQ30OKFSr +XEmawJdIW3DKeM9iSunxxwAILT68AnTWO+ex/8466ilXrNe5bm9GMiEzyOrnEFqS +p/sBjHohLVv18V7YnMbn1w4POWt2CEYINxUotpnj +-----END CERTIFICATE REQUEST----- diff --git a/charts/druid/resources/ca.csr.conf b/charts/druid/resources/ca.csr.conf new file mode 100644 index 000000000..4c9d5c6bc --- /dev/null +++ b/charts/druid/resources/ca.csr.conf @@ -0,0 +1,4 @@ +[ v3_ext ] +keyUsage=critical,digitalSignature,keyEncipherment,keyCertSign,cRLSign +basicConstraints=critical,CA:TRUE +subjectKeyIdentifier=hash diff --git a/charts/druid/resources/ca.key b/charts/druid/resources/ca.key new file mode 100644 index 000000000..5f640cea5 --- /dev/null +++ b/charts/druid/resources/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAyluLT/y9ublUuRt40NQmqE5EIlejDNU4gDGwX2w+kzDvtlWl +bQtI7PIZY6R+Y/aqv8xb8+wGKeXC8lkRCPlf3HtjSULVRI3buAwoV82vb586T9gD +78rJTwXWqrmdsnUPdyS00JQPcgwO17AoCIyt2ErrT3ZZr0Zl1RlQfhM74UZMn/1y +2j5AfQ2OhvzgfbyzdvMOHrrwx4z1nPlV3OSs7Xj7NC9P6xnpSU552qunFfL7Tsu7 +9485gYgp8V+LohrV41gFZceVXzcy5LhF+OwxPFHm+2ReY6vRlf2U9KiSO061PNs/ +CbIaX6nsf0/hMRXnmzsD6UMswFqOD8Ql7U53CQIDAQABAoIBAQCa8KUmxZpvjlw0 +r7g+DXLcA7FfqkKKHOh6H5GwPq1a/mlM7x8O80kPQ//0r4qm98Odv9fYWKwPgIFY +FqJYgLjJlSBcg/PP3d6SeZyaPj9J9F0trXS2MksZWpsXdbAtaDxDObPI0NVSubDT +rWm5wh2KMzMhULky33AxToS3Q/rhbRkzf8puS8jPvDWIUhslkcfIPDlU3jS51Hx3 +xX3Lwiya3icuz5rGz1+hPgfLFyd733n6AdYa0ZGr0V9ettM2FOlql1rUwAdr2lCh +1Vk7SnVJkwXoWzXyIWuJYJav3fXRvZJKCLI3w7AVHjA9tm0ru2ZFlGdJmPIHv+X2 +39B8l6wRAoGBAPE/Uhcdrlevc5Juc6OEYw6Q5xpQGFnZZIIL6ssYFxe7L96QxEJx +JUTK+7rEwT3+1bfxE4aDSTnvfBKtWJ5I4xDc7fxz3qZq7xHPWCcCOmDPI4LyGDyX +zX9ACgMEy6FheUOrTDE9WeD66zEf2k3myxM8Fg2OXVdsD3/jKYBJcalTAoGBANa7 +aYRa939dcSURyIq+t6upznraIFjQCCLbgqB5la/8IgsVdMIP2eCL93utVmi/k1ip +tub3ocjQDBMaM+8jwIlri4kEDp+YA9N6B+NKq0f3wqrUaxju3m+AuRBvTG78OM2K ++QDbrj0QBqM7gn+WPhXu0Csyom6zhUlw6oV/amazAoGBAJLwG2Li/CaCQeVlit+E +qgTcMzaupDZNl88xzP+Im8AjjiOEj1VcrvN4otx+LKlDBw8zMc5TNo2oFS0XEgvO +nybNOdLzDCDWruhtHucbj00ZvSprpGF7oHmqGq7+A702rnKN6ilvkcfbbNcN7JxY +P0Zm6ZO2K5oswkAFr2ho02brAoGBAIRu03XosInhtF1baWaAl0CNGqZUDu0XaF0v +KpIUSoheJRTW58qkAgjl6f1h+0/hD1v4V7B8+0+hWEETwQH099MI++bGFMrC82Ei ++uhobdiV6n53QZIACDmWlwyWx/oPHm1OD6JC2mQYlS9pDWPo4mOehM6PjPRfTY3s +0QcCfTnXAoGAH+CD5BRc1IsLSyQbSO8l9aIjXWaom9ND30Bu7crpTJhjkK51oAJK +QG0sSvAjmSCcZlVvYEkba6idaUHBQQzlawrakeaKAaifpjEXV0K2Hu3s7bl8KEDc +z379CzvhQcavr5VUFTCfK28iSzY/H9QrhL8iE094uQjnC3niZN5j5yQ= +-----END RSA PRIVATE KEY----- diff --git a/charts/druid/resources/ca.srl b/charts/druid/resources/ca.srl new file mode 100644 index 000000000..d4ab49e9a --- /dev/null +++ b/charts/druid/resources/ca.srl @@ -0,0 +1 @@ +C2DE1F0C92685E9D diff --git a/charts/druid/resources/server.crt b/charts/druid/resources/server.crt new file mode 100644 index 000000000..18711fdd6 --- /dev/null +++ b/charts/druid/resources/server.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDzzCCAregAwIBAgIJAMLeHwySaF6dMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV +BAMMCmV0Y2QtZHJ1aWQwHhcNMjQwMTE5MDczNDUzWhcNMzQwMTE5MDczNDUzWjAc +MRowGAYDVQQDDBFldGNkLWRydWlkLXNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANGaTM6yFIT6/lq4pyzs2dvcavVyY8iiwWrlQeksFnymYCwG +eRScs/1umdi7fus7MZUBXiyXlxjZ9HlQZNuwJ1tZuhu4QAjiHScXDpnjAl4aAiIo +X8hgZqctDQr5jfjW5ZPs6RXxCX5naDiqY4de4AdOx15l2RIrOYyQyEovhIYphTu1 +BXnwT6Avdxa+qsFQIpLkwyNSDxRUcGt/W0rnLkgX16oaW6X3z1nLNupmYCt+BNbb +p33vquTy4waTx0pmylX+oVdDnApyiG6SeQoJ9cnoq+t67gJNw37EUspfdB/71wvY +pBj4yGimVV4J13wXYxQcSeggSooigzZHmNLmO8UCAwEAAaOCARkwggEVMA4GA1Ud +DwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMB8G +A1UdIwQYMBaAFKlT6MBLcYwKDi5vWM3hx4NBNoFaMIG+BgNVHREEgbYwgbOCCmV0 +Y2QtZHJ1aWSCEmV0Y2QtZHJ1aWQuZGVmYXVsdIIWZXRjZC1kcnVpZC5kZWZhdWx0 +LnN2Y4IkZXRjZC1kcnVpZC5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsghNldGNk +LWRydWlkLmUyZS10ZXN0ghdldGNkLWRydWlkLmUyZS10ZXN0LnN2Y4IlZXRjZC1k +cnVpZC5lMmUtdGVzdC5zdmMuY2x1c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOC +AQEArxS38L7spkQRJ+QNOnMQD6NKsrG3bArzUW5pqSltSa4kDR2Eme0lm+UT9CrV +689+7edJmhwMWkRYTC0CGQooeal70ZyyX4nWP++ovhI/WBQjrHY++EpwNzgt8krB +BCq+XUnJj0vgIid665sm9HarJqK1dWXRwSJ4uULrLO7F+9TORbrY1jPGNXauCW49 +EBSb9J+woRhjHNOPGSEMrNGjGZtgUWUrdNQBvbuZohZ9IVXHhSQKYBG1wixSBrXU +C55gA1Og3BmtL6ZrRTQvekzYA4qr04Ln+a2jWhpkR1azQkkiMJy/7RaXSPMKrLrT +Ituc8tVxSDTQnTuUlgCztpylPA== +-----END CERTIFICATE----- diff --git a/charts/druid/resources/server.csr b/charts/druid/resources/server.csr new file mode 100644 index 000000000..ae0de131e --- /dev/null +++ b/charts/druid/resources/server.csr @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIDNzCCAh8CAQAwHDEaMBgGA1UEAwwRZXRjZC1kcnVpZC1zZXJ2ZXIwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRmkzOshSE+v5auKcs7Nnb3Gr1cmPI +osFq5UHpLBZ8pmAsBnkUnLP9bpnYu37rOzGVAV4sl5cY2fR5UGTbsCdbWbobuEAI +4h0nFw6Z4wJeGgIiKF/IYGanLQ0K+Y341uWT7OkV8Ql+Z2g4qmOHXuAHTsdeZdkS +KzmMkMhKL4SGKYU7tQV58E+gL3cWvqrBUCKS5MMjUg8UVHBrf1tK5y5IF9eqGlul +989ZyzbqZmArfgTW26d976rk8uMGk8dKZspV/qFXQ5wKcohuknkKCfXJ6Kvreu4C +TcN+xFLKX3Qf+9cL2KQY+MhoplVeCdd8F2MUHEnoIEqKIoM2R5jS5jvFAgMBAAGg +gdUwgdIGCSqGSIb3DQEJDjGBxDCBwTCBvgYDVR0RBIG2MIGzggpldGNkLWRydWlk +ghJldGNkLWRydWlkLmRlZmF1bHSCFmV0Y2QtZHJ1aWQuZGVmYXVsdC5zdmOCJGV0 +Y2QtZHJ1aWQuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbIITZXRjZC1kcnVpZC5l +MmUtdGVzdIIXZXRjZC1kcnVpZC5lMmUtdGVzdC5zdmOCJWV0Y2QtZHJ1aWQuZTJl +LXRlc3Quc3ZjLmNsdXN0ZXIubG9jYWwwDQYJKoZIhvcNAQELBQADggEBALGYEG4m +XdNZsnBUzXt1LJsxDpnJ4+iGLRWmfoGEdh4uLxt29ejKXMWU+PeP5o4zGrdLFj0x +Yc2QkQ3p+diiGlJ2SHk/saKgI7JeB2qIH2DGkdQK+PLq58JFnS2heeaN76xW/T6A +jRe3+lAco15np9ciFYC0oFyXMIBAeqzzraHEYsl/h/FErEMdwFEY7+PI7dOzFzPf +u39RGQO9/sWBGzYXXVvSd2zuACPjLg00R25kFqem839N89NJF992FM+9mjyzd9+u +2f6943QE3FSr6V2DNM+OawXdfpOPdQk1hfGRaNZbBB7LVVETdJFgb1LTz7pVKf4D +28HXkxpTBdRKvg0= +-----END CERTIFICATE REQUEST----- diff --git a/charts/druid/resources/server.csr.conf b/charts/druid/resources/server.csr.conf new file mode 100644 index 000000000..13f14af3a --- /dev/null +++ b/charts/druid/resources/server.csr.conf @@ -0,0 +1,28 @@ +[ req ] +default_bits = 2048 +prompt = no +default_md = sha256 +req_extensions = req_ext +distinguished_name = dn + +[ dn ] +CN = etcd-druid-server + +[ req_ext ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = etcd-druid +DNS.2 = etcd-druid.default +DNS.3 = etcd-druid.default.svc +DNS.4 = etcd-druid.default.svc.cluster.local +DNS.5 = etcd-druid.e2e-test +DNS.6 = etcd-druid.e2e-test.svc +DNS.7 = etcd-druid.e2e-test.svc.cluster.local + +[ v3_ext ] +keyUsage=critical,digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +basicConstraints=critical,CA:FALSE +authorityKeyIdentifier=keyid:always +subjectAltName=@alt_names diff --git a/charts/druid/resources/server.key b/charts/druid/resources/server.key new file mode 100644 index 000000000..44e74f093 --- /dev/null +++ b/charts/druid/resources/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA0ZpMzrIUhPr+WrinLOzZ29xq9XJjyKLBauVB6SwWfKZgLAZ5 +FJyz/W6Z2Lt+6zsxlQFeLJeXGNn0eVBk27AnW1m6G7hACOIdJxcOmeMCXhoCIihf +yGBmpy0NCvmN+Nblk+zpFfEJfmdoOKpjh17gB07HXmXZEis5jJDISi+EhimFO7UF +efBPoC93Fr6qwVAikuTDI1IPFFRwa39bSucuSBfXqhpbpffPWcs26mZgK34E1tun +fe+q5PLjBpPHSmbKVf6hV0OcCnKIbpJ5Cgn1yeir63ruAk3DfsRSyl90H/vXC9ik +GPjIaKZVXgnXfBdjFBxJ6CBKiiKDNkeY0uY7xQIDAQABAoIBAQCA+ah/m7biqSuw +mWVNUKUyE+zBAdWC1RudclrfZyCjLSGKlaUV2OMsdE04vw73ImJ9JO1VFSaFxUAZ +Ei4tvLRLEuhhPDwy+ygjMJ3sdXK0gt6DNELUIvWnsyIOeccxstJr9uDqfnpwDKNm +EvrRhsDl3kTxwkwTUQPFiiWk+J5cirzYMgSulRGr8Q9ATWS8crxkjoBuht+IzBkj +g+m8qoh0QsH4iopB39ncyN+eJMT/gyikLeyZYlrCHywC6VDdmFEZJMhJxEcRhdqK +WjjT5MIHXya0+2dDtApExV2EO5KUaJesY3oovquEeGKjM0veMc3nXaxGobUzgbwz +JHRiAr8BAoGBAO8ErIJKb9fh/FwA3lk+fv2IIochgP8RWlDFt06/0d13jNJlnnDu +25jqnZQU/pHOuyHAX3zJ/8vINphW3ZlDdwQwkGuHq3Vrx1obNfzn0jxXTw0ny8FU +IrzYPJhu7PUKgA2N6Op3fItSqELF+YuSaF7UkBwNBfdlXbcuu60PUrpBAoGBAOB+ +m+dMNaQae9W2COoOlIcUjfU03b1EJIXkgbkc4oLTiBr2o+4j/6D4ceuJ0O+oB530 +b8H2qycDorOh+OZYSVKDN9lAh6WW6WrY3qhxVJO4VTZmlRAu2tsfbQHIRBFMx+Gb +xyUB8IJCo6mJFsIrJbtbtEApcEQ3xn5pd+BsXXiFAoGANPslPhA/8GuQY9hxcPjQ +b2SAutFrZZ60FVluQ49zdpLsbSrHTyMzHBdNbKimsS1Q/69Du0aPY91ydJw0rKdu +t0gIiz1cfT+xxcBoXDIKwQNda34ZxTXgKVoJPITCE6MLNXXyXh60RbHByGNX6F9A +sVyOwRkF7IvUlSlPwpG19cECgYB1BUpbTTyA6FU414TaDPObXjmJYh9b81Nef/im +98mTDJTcCUj2b8lmrVskvlgtNNeTnMFMyDn7Qkfjf0DxQYAnBMNt+dc7l+jgi1+1 +Si4eOm4gx2RYZTQsGLuHvE7O/ckLb1pljhdCr14El7NHT5qBiZHyCCe/R5uudWfJ +knM+yQKBgQCYAlKi+gQh8GqYPhQO1JjigbXlM8TVff6F1Ge8wHQVWjIwPhs/auxM +5mt8EGKHs+JuUEkbr3hbvNJDuomxADVwmTtWjZnk1BUXC1pYLD9RvSQVmeQrI8Ve +WC2K9FWr2CQcW1wOEQIjvJBfx6Zp/Iu5upiRm1apMJSFRvl8Z+JWlQ== +-----END RSA PRIVATE KEY----- diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index dd33b0d11..51243fee9 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -2,9 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Release.Name }} + name: etcd-druid namespace: {{ .Release.Namespace }} labels: + gardener.cloud/role: etcd-druid spec: replicas: {{ .Values.replicas }} selector: @@ -24,8 +25,7 @@ spec: - /etcd-druid - --enable-leader-election=true - --ignore-operation-annotation={{ .Values.ignoreOperationAnnotation }} - - --workers=3 - - --custodian-sync-period=15s + - --etcd-workers=3 {{- if .Values.enableBackupCompaction }} - --enable-backup-compaction={{ .Values.enableBackupCompaction }} {{- end }} @@ -43,6 +43,21 @@ spec: {{- end }} - --feature-gates={{ $featuregates | trimSuffix "," }} {{- end }} + + {{- if (((.Values.server).webhook).bindAddress) }} + - --webhook-server-bind-address={{ .Values.server.webhook.bindAddress }} + {{- end }} + {{- if (((.Values.server).webhook).port) }} + - --webhook-server-port={{ .Values.server.webhook.port }} + {{- end }} + {{- if ((((.Values.server).webhook).tls).serverCertDir) }} + - --webhook-server-tls-server-cert-dir={{ .Values.server.webhook.tls.serverCertDir }} + {{- end }} + + {{- if .Values.enableSentinelWebhook }} + - --enable-sentinel-webhook={{ .Values.enableSentinelWebhook }} + {{- end }} + resources: limits: cpu: 300m @@ -52,3 +67,12 @@ spec: memory: 128Mi ports: - containerPort: 9569 + volumeMounts: + - mountPath: /etc/webhook-server-tls + name: tls + readOnly: true + volumes: + - name: tls + secret: + defaultMode: 420 + secretName: etcd-druid-server-tls diff --git a/charts/druid/templates/secret-ca-crt.yaml b/charts/druid/templates/secret-ca-crt.yaml new file mode 100644 index 000000000..ea1096858 --- /dev/null +++ b/charts/druid/templates/secret-ca-crt.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ca-etcd-druid + namespace: {{ .Release.Namespace }} + labels: + gardener.cloud/role: etcd-druid +type: Opaque +data: + bundle.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3ekNDQWRlZ0F3SUJBZ0lKQU51a3lQUVVxYnhNTUEwR0NTcUdTSWIzRFFFQkN3VUFNQlV4RXpBUkJnTlYKQkFNTUNtVjBZMlF0WkhKMWFXUXdIaGNOTWpRd01URTRNVFV6TVRJeldoY05NelF3TVRFNE1UVXpNVEl6V2pBVgpNUk13RVFZRFZRUUREQXBsZEdOa0xXUnlkV2xrTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCCkNnS0NBUUVBeWx1TFQveTl1YmxVdVJ0NDBOUW1xRTVFSWxlakROVTRnREd3WDJ3K2t6RHZ0bFdsYlF0STdQSVoKWTZSK1kvYXF2OHhiOCt3R0tlWEM4bGtSQ1BsZjNIdGpTVUxWUkkzYnVBd29WODJ2YjU4NlQ5Z0Q3OHJKVHdYVwpxcm1kc25VUGR5UzAwSlFQY2d3TzE3QW9DSXl0MkVyclQzWlpyMFpsMVJsUWZoTTc0VVpNbi8xeTJqNUFmUTJPCmh2emdmYnl6ZHZNT0hycnd4NHoxblBsVjNPU3M3WGo3TkM5UDZ4bnBTVTU1MnF1bkZmTDdUc3U3OTQ4NWdZZ3AKOFYrTG9oclY0MWdGWmNlVlh6Y3k1TGhGK093eFBGSG0rMlJlWTZ2UmxmMlU5S2lTTzA2MVBOcy9DYklhWDZucwpmMC9oTVJYbm16c0Q2VU1zd0ZxT0Q4UWw3VTUzQ1FJREFRQUJvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBYVl3CkR3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXFWUG93RXR4akFvT0xtOVl6ZUhIZzBFMmdWb3cKRFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUR3ZGduT25POStWNmNvdTQ4Z1czeUtONVhMNmJkRm5mbng5aldlTQpkelBsc2tBa0hKTzAvTXpqbk1DUnFxQmREajM2Yzcvb3lJcEpBall1ZmFrMXJDcW9rVEpHOVNzTlE4VlJmT0FSCkRRU0o0YmsrMzNQT29MSnNrZTNzdm9ZM0RPc3h4RzZNT0o5S29aRTFaeWxEd3hrYVo0U212b1NmTStmUHU3aloKenA1RWJXVnE0YmJUYjgrRnZqSDQ5bmY1eGY1WHJLbTJ1Z3dZZWIwWnpnYkIrSGdNSHRvMGtWcXFObVlrbDYyMwp0NFBOV3Q1ck9sY2swd1dKeVVZZHdONk1qQWQrZVZaQmJRRzQ3TWpZNTJTenBwbGNQdVJuaU81Q0tKb0JZdmU1CmZzTUhIR0dXeEg0b2JCNk9WZytIM0pJNGpRWndTUXZGRVBKcm5LNXV0RjdySmw0PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== diff --git a/charts/druid/templates/secret-server-tls-crt.yaml b/charts/druid/templates/secret-server-tls-crt.yaml new file mode 100644 index 000000000..e4898e247 --- /dev/null +++ b/charts/druid/templates/secret-server-tls-crt.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-druid-server-tls + namespace: {{ .Release.Namespace }} + labels: + gardener.cloud/role: etcd-druid +type: kubernetes.io/tls +data: +# TODO: generate TLS server cert that can match etcd-druid.*.svc, since the helm chart can be deployed to any namespace +# Currently the only DNS names used are etcd-druid,etcd-druid.[default|e2e-test].svc,etcd-druid.[default|e2e-test].svc.cluster.local + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR6ekNDQXJlZ0F3SUJBZ0lKQU1MZUh3eVNhRjZkTUEwR0NTcUdTSWIzRFFFQkN3VUFNQlV4RXpBUkJnTlYKQkFNTUNtVjBZMlF0WkhKMWFXUXdIaGNOTWpRd01URTVNRGN6TkRVeldoY05NelF3TVRFNU1EY3pORFV6V2pBYwpNUm93R0FZRFZRUUREQkZsZEdOa0xXUnlkV2xrTFhObGNuWmxjakNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFECmdnRVBBRENDQVFvQ2dnRUJBTkdhVE02eUZJVDYvbHE0cHl6czJkdmNhdlZ5WThpaXdXcmxRZWtzRm55bVlDd0cKZVJTY3MvMXVtZGk3ZnVzN01aVUJYaXlYbHhqWjlIbFFaTnV3SjF0WnVodTRRQWppSFNjWERwbmpBbDRhQWlJbwpYOGhnWnFjdERRcjVqZmpXNVpQczZSWHhDWDVuYURpcVk0ZGU0QWRPeDE1bDJSSXJPWXlReUVvdmhJWXBoVHUxCkJYbndUNkF2ZHhhK3FzRlFJcExrd3lOU0R4UlVjR3QvVzBybkxrZ1gxNm9hVzZYM3oxbkxOdXBtWUN0K0JOYmIKcDMzdnF1VHk0d2FUeDBwbXlsWCtvVmREbkFweWlHNlNlUW9KOWNub3ErdDY3Z0pOdzM3RVVzcGZkQi83MXd2WQpwQmo0eUdpbVZWNEoxM3dYWXhRY1NlZ2dTb29pZ3paSG1OTG1POFVDQXdFQUFhT0NBUmt3Z2dFVk1BNEdBMVVkCkR3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBVEFNQmdOVkhSTUJBZjhFQWpBQU1COEcKQTFVZEl3UVlNQmFBRktsVDZNQkxjWXdLRGk1dldNM2h4NE5CTm9GYU1JRytCZ05WSFJFRWdiWXdnYk9DQ21WMApZMlF0WkhKMWFXU0NFbVYwWTJRdFpISjFhV1F1WkdWbVlYVnNkSUlXWlhSalpDMWtjblZwWkM1a1pXWmhkV3gwCkxuTjJZNElrWlhSalpDMWtjblZwWkM1a1pXWmhkV3gwTG5OMll5NWpiSFZ6ZEdWeUxteHZZMkZzZ2hObGRHTmsKTFdSeWRXbGtMbVV5WlMxMFpYTjBnaGRsZEdOa0xXUnlkV2xrTG1VeVpTMTBaWE4wTG5OMlk0SWxaWFJqWkMxawpjblZwWkM1bE1tVXRkR1Z6ZEM1emRtTXVZMngxYzNSbGNpNXNiMk5oYkRBTkJna3Foa2lHOXcwQkFRc0ZBQU9DCkFRRUFyeFMzOEw3c3BrUVJKK1FOT25NUUQ2TktzckczYkFyelVXNXBxU2x0U2E0a0RSMkVtZTBsbStVVDlDclYKNjg5KzdlZEptaHdNV2tSWVRDMENHUW9vZWFsNzBaeXlYNG5XUCsrb3ZoSS9XQlFqckhZKytFcHdOemd0OGtyQgpCQ3ErWFVuSmowdmdJaWQ2NjVzbTlIYXJKcUsxZFdYUndTSjR1VUxyTE83Ris5VE9SYnJZMWpQR05YYXVDVzQ5CkVCU2I5Sit3b1JoakhOT1BHU0VNck5HakdadGdVV1VyZE5RQnZidVpvaFo5SVZYSGhTUUtZQkcxd2l4U0JyWFUKQzU1Z0ExT2czQm10TDZaclJUUXZla3pZQTRxcjA0TG4rYTJqV2hwa1IxYXpRa2tpTUp5LzdSYVhTUE1LckxyVApJdHVjOHRWeFNEVFFuVHVVbGdDenRweWxQQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBMFpwTXpySVVoUHIrV3JpbkxPeloyOXhxOVhKanlLTEJhdVZCNlN3V2ZLWmdMQVo1CkZKeXovVzZaMkx0KzZ6c3hsUUZlTEplWEdObjBlVkJrMjdBblcxbTZHN2hBQ09JZEp4Y09tZU1DWGhvQ0lpaGYKeUdCbXB5ME5Ddm1OK05ibGsrenBGZkVKZm1kb09LcGpoMTdnQjA3SFhtWFpFaXM1akpESVNpK0VoaW1GTzdVRgplZkJQb0M5M0ZyNnF3VkFpa3VUREkxSVBGRlJ3YTM5YlN1Y3VTQmZYcWhwYnBmZlBXY3MyNm1aZ0szNEUxdHVuCmZlK3E1UExqQnBQSFNtYktWZjZoVjBPY0NuS0licEo1Q2duMXllaXI2M3J1QWszRGZzUlN5bDkwSC92WEM5aWsKR1BqSWFLWlZYZ25YZkJkakZCeEo2Q0JLaWlLRE5rZVkwdVk3eFFJREFRQUJBb0lCQVFDQSthaC9tN2JpcVN1dwptV1ZOVUtVeUUrekJBZFdDMVJ1ZGNscmZaeUNqTFNHS2xhVVYyT01zZEUwNHZ3NzNJbUo5Sk8xVkZTYUZ4VUFaCkVpNHR2TFJMRXVoaFBEd3kreWdqTUozc2RYSzBndDZETkVMVUl2V25zeUlPZWNjeHN0SnI5dURxZm5wd0RLTm0KRXZyUmhzRGwza1R4d2t3VFVRUEZpaVdrK0o1Y2lyellNZ1N1bFJHcjhROUFUV1M4Y3J4a2pvQnVodCtJekJragpnK204cW9oMFFzSDRpb3BCMzluY3lOK2VKTVQvZ3lpa0xleVpZbHJDSHl3QzZWRGRtRkVaSk1oSnhFY1JoZHFLCldqalQ1TUlIWHlhMCsyZER0QXBFeFYyRU81S1VhSmVzWTNvb3ZxdUVlR0tqTTB2ZU1jM25YYXhHb2JVemdid3oKSkhSaUFyOEJBb0dCQU84RXJJSktiOWZoL0Z3QTNsaytmdjJJSW9jaGdQOFJXbERGdDA2LzBkMTNqTkpsbm5EdQoyNWpxblpRVS9wSE91eUhBWDN6Si84dklOcGhXM1psRGR3UXdrR3VIcTNWcngxb2JOZnpuMGp4WFR3MG55OEZVCklyellQSmh1N1BVS2dBMk42T3AzZkl0U3FFTEYrWXVTYUY3VWtCd05CZmRsWGJjdXU2MFBVcnBCQW9HQkFPQisKbStkTU5hUWFlOVcyQ09vT2xJY1VqZlUwM2IxRUpJWGtnYmtjNG9MVGlCcjJvKzRqLzZENGNldUowTytvQjUzMApiOEgycXljRG9yT2grT1pZU1ZLRE45bEFoNldXNldyWTNxaHhWSk80VlRabWxSQXUydHNmYlFISVJCRk14K0diCnh5VUI4SUpDbzZtSkZzSXJKYnRidEVBcGNFUTN4bjVwZCtCc1hYaUZBb0dBTlBzbFBoQS84R3VRWTloeGNQalEKYjJTQXV0RnJaWjYwRlZsdVE0OXpkcExzYlNySFR5TXpIQmROYktpbXNTMVEvNjlEdTBhUFk5MXlkSncwcktkdQp0MGdJaXoxY2ZUK3h4Y0JvWERJS3dRTmRhMzRaeFRYZ0tWb0pQSVRDRTZNTE5YWHlYaDYwUmJIQnlHTlg2RjlBCnNWeU93UmtGN0l2VWxTbFB3cEcxOWNFQ2dZQjFCVXBiVFR5QTZGVTQxNFRhRFBPYlhqbUpZaDliODFOZWYvaW0KOThtVERKVGNDVWoyYjhsbXJWc2t2bGd0Tk5lVG5NRk15RG43UWtmamYwRHhRWUFuQk1OdCtkYzdsK2pnaTErMQpTaTRlT200Z3gyUllaVFFzR0x1SHZFN08vY2tMYjFwbGpoZENyMTRFbDdOSFQ1cUJpWkh5Q0NlL1I1dXVkV2ZKCmtuTSt5UUtCZ1FDWUFsS2krZ1FoOEdxWVBoUU8xSmppZ2JYbE04VFZmZjZGMUdlOHdIUVZXakl3UGhzL2F1eE0KNW10OEVHS0hzK0p1VUVrYnIzaGJ2TkpEdW9teEFEVndtVHRXalpuazFCVVhDMXBZTEQ5UnZTUVZtZVFySThWZQpXQzJLOUZXcjJDUWNXMXdPRVFJanZKQmZ4NlpwL0l1NXVwaVJtMWFwTUpTRlJ2bDhaK0pXbFE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/charts/druid/templates/service.yaml b/charts/druid/templates/service.yaml new file mode 100644 index 000000000..82b752505 --- /dev/null +++ b/charts/druid/templates/service.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + gardener.cloud/role: etcd-druid + name: etcd-druid + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + gardener.cloud/role: etcd-druid + ports: + - name: metrics + port: 8080 + protocol: TCP + targetPort: 8080 + - name: webhooks + port: 9443 + protocol: TCP + targetPort: 9443 diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml new file mode 100644 index 000000000..a2093bcc0 --- /dev/null +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: etcd-druid + namespace: {{ .Release.Namespace }} + labels: + gardener.cloud/role: etcd-druid +webhooks: + - admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3ekNDQWRlZ0F3SUJBZ0lKQU51a3lQUVVxYnhNTUEwR0NTcUdTSWIzRFFFQkN3VUFNQlV4RXpBUkJnTlYKQkFNTUNtVjBZMlF0WkhKMWFXUXdIaGNOTWpRd01URTRNVFV6TVRJeldoY05NelF3TVRFNE1UVXpNVEl6V2pBVgpNUk13RVFZRFZRUUREQXBsZEdOa0xXUnlkV2xrTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCCkNnS0NBUUVBeWx1TFQveTl1YmxVdVJ0NDBOUW1xRTVFSWxlakROVTRnREd3WDJ3K2t6RHZ0bFdsYlF0STdQSVoKWTZSK1kvYXF2OHhiOCt3R0tlWEM4bGtSQ1BsZjNIdGpTVUxWUkkzYnVBd29WODJ2YjU4NlQ5Z0Q3OHJKVHdYVwpxcm1kc25VUGR5UzAwSlFQY2d3TzE3QW9DSXl0MkVyclQzWlpyMFpsMVJsUWZoTTc0VVpNbi8xeTJqNUFmUTJPCmh2emdmYnl6ZHZNT0hycnd4NHoxblBsVjNPU3M3WGo3TkM5UDZ4bnBTVTU1MnF1bkZmTDdUc3U3OTQ4NWdZZ3AKOFYrTG9oclY0MWdGWmNlVlh6Y3k1TGhGK093eFBGSG0rMlJlWTZ2UmxmMlU5S2lTTzA2MVBOcy9DYklhWDZucwpmMC9oTVJYbm16c0Q2VU1zd0ZxT0Q4UWw3VTUzQ1FJREFRQUJvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBYVl3CkR3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXFWUG93RXR4akFvT0xtOVl6ZUhIZzBFMmdWb3cKRFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUR3ZGduT25POStWNmNvdTQ4Z1czeUtONVhMNmJkRm5mbng5aldlTQpkelBsc2tBa0hKTzAvTXpqbk1DUnFxQmREajM2Yzcvb3lJcEpBall1ZmFrMXJDcW9rVEpHOVNzTlE4VlJmT0FSCkRRU0o0YmsrMzNQT29MSnNrZTNzdm9ZM0RPc3h4RzZNT0o5S29aRTFaeWxEd3hrYVo0U212b1NmTStmUHU3aloKenA1RWJXVnE0YmJUYjgrRnZqSDQ5bmY1eGY1WHJLbTJ1Z3dZZWIwWnpnYkIrSGdNSHRvMGtWcXFObVlrbDYyMwp0NFBOV3Q1ck9sY2swd1dKeVVZZHdONk1qQWQrZVZaQmJRRzQ3TWpZNTJTenBwbGNQdVJuaU81Q0tKb0JZdmU1CmZzTUhIR0dXeEg0b2JCNk9WZytIM0pJNGpRWndTUXZGRVBKcm5LNXV0RjdySmw0PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: etcd-druid + namespace: {{ .Release.Namespace }} + path: /webhooks/sentinel + port: 9443 + failurePolicy: Fail + matchPolicy: Exact + name: sentinel.webhooks.druid.gardener.cloud + namespaceSelector: {} + objectSelector: + matchLabels: + app.kubernetes.io/managed-by: etcd-druid + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - serviceaccounts + - services + - configmaps + scope: '*' + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - roles + - rolebindings + scope: '*' + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - statefulsets + scope: '*' + - apiGroups: + - policy + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - poddisruptionbudgets + scope: '*' + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - jobs + scope: '*' + - apiGroups: + - coordination.k8s.io + apiVersions: + - v1 + operations: + - DELETE + resources: + - leases + scope: '*' + sideEffects: None + timeoutSeconds: 10 diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index 1d82906b3..c50f523a5 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -11,3 +11,12 @@ ignoreOperationAnnotation: false # metricsScrapeWaitDuration: "10s" # featureGates: # UseEtcdWrapper: true + +server: + webhook: + bindAddress: "" + port: 9443 + tls: + serverCertDir: /etc/webhook-server-tls + +enableSentinelWebhook: true diff --git a/internal/controller/config.go b/internal/controller/config.go index 7f9b41d06..8a3f72322 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -23,6 +23,8 @@ import ( "github.com/gardener/etcd-druid/internal/controller/secret" "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" + "github.com/gardener/etcd-druid/internal/webhook/sentinel" + flag "github.com/spf13/pflag" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/component-base/featuregate" @@ -30,18 +32,55 @@ import ( const ( metricsAddrFlagName = "metrics-addr" + webhookServerBindAddressFlagName = "webhook-server-bind-address" + webhookServerPortFlagName = "webhook-server-port" + webhookServerTLSServerCertDir = "webhook-server-tls-server-cert-dir" enableLeaderElectionFlagName = "enable-leader-election" leaderElectionIDFlagName = "leader-election-id" leaderElectionResourceLockFlagName = "leader-election-resource-lock" disableLeaseCacheFlagName = "disable-lease-cache" defaultMetricsAddr = ":8080" + defaultWebhookServerBindAddress = "" + defaultWebhookServerPort = 9443 + defaultWebhookServerTLSServerCert = "/etc/webhook-server-tls" defaultEnableLeaderElection = false defaultLeaderElectionID = "druid-leader-election" defaultLeaderElectionResourceLock = resourcelock.LeasesResourceLock defaultDisableLeaseCache = false ) +// ServerConfig contains details for the HTTP(S) servers. +type ServerConfig struct { + // Webhook is the configuration for the HTTPS webhook server. + Webhook HTTPSServer + // Metrics is the configuration for serving the metrics endpoint. + Metrics *Server +} + +// HTTPSServer is the configuration for the HTTPSServer server. +type HTTPSServer struct { + // Server is the configuration for the bind address and the port. + Server + // TLSServer contains information about the TLS configuration for an HTTPS server. + TLS TLSServer +} + +// TLSServer contains information about the TLS configuration for an HTTPS server. +type TLSServer struct { + // ServerCertDir is the path to a directory containing the server's TLS certificate and key (the files must be + // named tls.crt and tls.key respectively). + ServerCertDir string +} + +// Server contains information for HTTP(S) server configuration. +type Server struct { + // BindAddress is the IP address on which to listen for the specified port. + BindAddress string + // Port is the port on which to serve unsecured, unauthenticated access. + Port int +} + // LeaderElectionConfig defines the configuration for the leader election for the controller manager. type LeaderElectionConfig struct { // EnableLeaderElection specifies whether to enable leader election for controller manager. @@ -57,7 +96,10 @@ type LeaderElectionConfig struct { // ManagerConfig defines the configuration for the controller manager. type ManagerConfig struct { // MetricsAddr is the address the metric endpoint binds to. + // Deprecated: This field will be eventually removed. Please use Server.Metrics.BindAddress instead. MetricsAddr string + // Server is the configuration for the HTTP server. + Server *ServerConfig LeaderElectionConfig // DisableLeaseCache specifies whether to disable cache for lease.coordination.k8s.io resources. DisableLeaseCache bool @@ -71,12 +113,25 @@ type ManagerConfig struct { EtcdCopyBackupsTaskControllerConfig *etcdcopybackupstask.Config // SecretControllerConfig is the configuration required for secret controller. SecretControllerConfig *secret.Config + // SentinelWebhookConfig is the configuration required for sentinel webhook. + SentinelWebhookConfig *sentinel.Config } // InitFromFlags initializes the controller manager config from the provided CLI flag set. func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { - flag.StringVar(&cfg.MetricsAddr, metricsAddrFlagName, defaultMetricsAddr, ""+ + cfg.Server = &ServerConfig{} + cfg.Server.Metrics = &Server{} + cfg.Server.Webhook = HTTPSServer{} + cfg.Server.Webhook.Server = Server{} + + flag.StringVar(&cfg.Server.Metrics.BindAddress, metricsAddrFlagName, defaultMetricsAddr, "The address the metric endpoint binds to.") + flag.StringVar(&cfg.Server.Webhook.Server.BindAddress, webhookServerBindAddressFlagName, defaultWebhookServerBindAddress, + "The IP address on which to listen for the HTTPS webhook server.") + flag.IntVar(&cfg.Server.Webhook.Server.Port, webhookServerPortFlagName, defaultWebhookServerPort, + "The port on which to listen for the HTTPS webhook server.") + flag.StringVar(&cfg.Server.Webhook.TLS.ServerCertDir, webhookServerTLSServerCertDir, defaultWebhookServerTLSServerCert, + "The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively).") flag.BoolVar(&cfg.EnableLeaderElection, enableLeaderElectionFlagName, defaultEnableLeaderElection, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") flag.StringVar(&cfg.LeaderElectionID, leaderElectionIDFlagName, defaultLeaderElectionID, @@ -102,6 +157,9 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { cfg.SecretControllerConfig = &secret.Config{} secret.InitFromFlags(fs, cfg.SecretControllerConfig) + cfg.SentinelWebhookConfig = &sentinel.Config{} + sentinel.InitFromFlags(fs, cfg.SentinelWebhookConfig) + return nil } @@ -137,6 +195,16 @@ func (cfg *ManagerConfig) Validate() error { if err := utils.ShouldBeOneOfAllowedValues("LeaderElectionResourceLock", getAllowedLeaderElectionResourceLocks(), cfg.LeaderElectionResourceLock); err != nil { return err } + + if cfg.SentinelWebhookConfig.Enabled { + if cfg.Server.Webhook.Port == 0 { + return fmt.Errorf("webhook port cannot be 0") + } + if cfg.Server.Webhook.TLS.ServerCertDir == "" { + return fmt.Errorf("webhook server cert dir cannot be empty") + } + } + if err := cfg.EtcdControllerConfig.Validate(); err != nil { return err } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 91e79a780..c852822c4 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -41,6 +41,7 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx resource.OperatorContext, etcd return r.recordIncompleteReconcileOperation(ctx, etcdObjectKey, stepResult) } } + ctx.Logger.Info("Finished spec reconciliation flow") return ctrlutils.ContinueReconcile() } diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 4c2da9628..128c2ce69 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -205,6 +205,7 @@ func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey return utils.ReconcileWithError(err) } + sLog.Info("Finished etcd status update") return utils.ContinueReconcile() } diff --git a/internal/controller/manager.go b/internal/controller/manager.go index fe02afcd6..e20d42282 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -23,6 +23,8 @@ import ( "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" + "github.com/gardener/etcd-druid/internal/webhook/sentinel" + coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" corev1 "k8s.io/api/core/v1" @@ -71,7 +73,10 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { return ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ ClientDisableCacheFor: uncachedObjects, Scheme: kubernetes.Scheme, - MetricsBindAddress: config.MetricsAddr, + MetricsBindAddress: config.Server.Metrics.BindAddress, + Host: config.Server.Webhook.BindAddress, + Port: config.Server.Webhook.Port, + CertDir: config.Server.Webhook.TLS.ServerCertDir, LeaderElection: config.EnableLeaderElection, LeaderElectionID: config.LeaderElectionID, LeaderElectionResourceLock: config.LeaderElectionResourceLock, @@ -113,8 +118,24 @@ func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) err // Add secret reconciler to the manager ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() - return secret.NewReconciler( + if err = secret.NewReconciler( mgr, config.SecretControllerConfig, - ).RegisterWithManager(ctx, mgr) + ).RegisterWithManager(ctx, mgr); err != nil { + return err + } + + // Add sentinel webhook to the manager + if config.SentinelWebhookConfig.Enabled { + var sentinelWebhook *sentinel.Handler + if sentinelWebhook, err = sentinel.NewHandler( + mgr, + config.SentinelWebhookConfig, + ); err != nil { + return err + } + return sentinelWebhook.RegisterWithManager(mgr) + } + + return nil } diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/sentinel/config.go new file mode 100644 index 000000000..def8746a3 --- /dev/null +++ b/internal/webhook/sentinel/config.go @@ -0,0 +1,21 @@ +package sentinel + +import flag "github.com/spf13/pflag" + +const ( + enableSentinelWebhookFlagName = "enable-sentinel-webhook" + + defaultEnableSentinelWebhook = false +) + +// Config defines the configuration for the Sentinel Webhook. +type Config struct { + // Enabled indicates whether the Sentinel Webhook is enabled. + Enabled bool +} + +// InitFromFlags initializes the config from the provided CLI flag set. +func InitFromFlags(fs *flag.FlagSet, cfg *Config) { + fs.BoolVar(&cfg.Enabled, enableSentinelWebhookFlagName, defaultEnableSentinelWebhook, + "Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid.") +} diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go new file mode 100644 index 000000000..82b792588 --- /dev/null +++ b/internal/webhook/sentinel/handler.go @@ -0,0 +1,243 @@ +package sentinel + +import ( + "context" + "fmt" + "net/http" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + "github.com/go-logr/logr" + admissionv1 "k8s.io/api/admission/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// Handler handles admission requests and prevents changes to resources created by etcd-druid. +type Handler struct { + client.Client + config *Config + decoder *admission.Decoder + logger logr.Logger +} + +// NewHandler creates a new handler for Sentinel Webhook. +func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { + decoder, err := admission.NewDecoder(mgr.GetScheme()) + if err != nil { + return nil, err + } + + return &Handler{ + Client: mgr.GetClient(), + config: config, + decoder: decoder, + logger: log.Log.WithName(handlerName), + }, nil +} + +func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { + var ( + requestGK = schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} + decodeOldObject bool + obj client.Object + err error + ) + + log := h.logger.WithValues("resourceKind", req.Kind.Kind, "name", req.Name, "namespace", req.Namespace, "operation", req.Operation) + log.Info("Sentinel webhook invoked") + + if req.Operation == admissionv1.Delete { + decodeOldObject = true + } + + switch requestGK { + case corev1.SchemeGroupVersion.WithKind("ServiceAccount").GroupKind(): + obj, err = h.decodeServiceAccount(req, decodeOldObject) + case corev1.SchemeGroupVersion.WithKind("Service").GroupKind(): + obj, err = h.decodeService(req, decodeOldObject) + case corev1.SchemeGroupVersion.WithKind("ConfigMap").GroupKind(): + obj, err = h.decodeConfigMap(req, decodeOldObject) + case rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(): + obj, err = h.decodeRole(req, decodeOldObject) + case rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(): + obj, err = h.decodeRoleBinding(req, decodeOldObject) + case appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(): + obj, err = h.decodeStatefulSet(req, decodeOldObject) + case policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(): + obj, err = h.decodePodDisruptionBudget(req, decodeOldObject) + case batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(): + obj, err = h.decodeJob(req, decodeOldObject) + case coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): + obj, err = h.decodeLease(req, decodeOldObject) + default: + return admission.Allowed(fmt.Sprintf("unexpected resource type: %s", requestGK)) + } + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + + etcdName, hasLabel := obj.GetLabels()[druidv1alpha1.LabelPartOfKey] + if !hasLabel { + return admission.Allowed(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey)) + } + + etcd := &druidv1alpha1.Etcd{} + if err := h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { + if apierrors.IsNotFound(err) { + return admission.Allowed(fmt.Sprintf("corresponding etcd %s not found", etcdName)) + } + return admission.Errored(http.StatusInternalServerError, err) + } + + // allow operations on resources if any etcd operation is currently under processing + if etcd.Status.LastOperation != nil && etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateProcessing { + return admission.Allowed(fmt.Sprintf("ongoing processing of etcd %s by etcd-druid requires changes to resources", etcd.Name)) + } + + if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.ResourceProtectionAnnotation) { + if etcd.GetAnnotations()[druidv1alpha1.ResourceProtectionAnnotation] == "false" { + return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) + } + } + + return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) +} + +func (h *Handler) decodeServiceAccount(req admission.Request, decodeOldObject bool) (client.Object, error) { + sa := &corev1.ServiceAccount{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, sa); err != nil { + return nil, err + } + return sa, nil + } + if err := h.decoder.Decode(req, sa); err != nil { + return nil, err + } + return sa, nil +} + +func (h *Handler) decodeService(req admission.Request, decodeOldObject bool) (client.Object, error) { + svc := &corev1.Service{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, svc); err != nil { + return nil, err + } + return svc, nil + } + if err := h.decoder.Decode(req, svc); err != nil { + return nil, err + } + return svc, nil +} + +func (h *Handler) decodeConfigMap(req admission.Request, decodeOldObject bool) (client.Object, error) { + cm := &corev1.ConfigMap{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, cm); err != nil { + return nil, err + } + return cm, nil + } + if err := h.decoder.Decode(req, cm); err != nil { + return nil, err + } + return cm, nil +} + +func (h *Handler) decodeRole(req admission.Request, decodeOldObject bool) (client.Object, error) { + role := &rbacv1.Role{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, role); err != nil { + return nil, err + } + return role, nil + } + if err := h.decoder.Decode(req, role); err != nil { + return nil, err + } + return role, nil +} + +func (h *Handler) decodeRoleBinding(req admission.Request, decodeOldObject bool) (client.Object, error) { + rb := &rbacv1.RoleBinding{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, rb); err != nil { + return nil, err + } + return rb, nil + } + if err := h.decoder.Decode(req, rb); err != nil { + return nil, err + } + return rb, nil +} + +func (h *Handler) decodeStatefulSet(req admission.Request, decodeOldObject bool) (client.Object, error) { + sts := &appsv1.StatefulSet{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, sts); err != nil { + return nil, err + } + return sts, nil + } + if err := h.decoder.Decode(req, sts); err != nil { + return nil, err + } + return sts, nil +} + +func (h *Handler) decodePodDisruptionBudget(req admission.Request, decodeOldObject bool) (client.Object, error) { + pdb := &policyv1.PodDisruptionBudget{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, pdb); err != nil { + return nil, err + } + return pdb, nil + } + if err := h.decoder.Decode(req, pdb); err != nil { + return nil, err + } + return pdb, nil +} + +func (h *Handler) decodeJob(req admission.Request, decodeOldObject bool) (client.Object, error) { + job := &batchv1.Job{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, job); err != nil { + return nil, err + } + return job, nil + } + if err := h.decoder.Decode(req, job); err != nil { + return nil, err + } + return job, nil +} + +func (h *Handler) decodeLease(req admission.Request, decodeOldObject bool) (client.Object, error) { + lease := &coordinationv1.Lease{} + if decodeOldObject { + if err := h.decoder.DecodeRaw(req.OldObject, lease); err != nil { + return nil, err + } + return lease, nil + } + if err := h.decoder.Decode(req, lease); err != nil { + return nil, err + } + return lease, nil +} diff --git a/internal/webhook/sentinel/register.go b/internal/webhook/sentinel/register.go new file mode 100644 index 000000000..bae075786 --- /dev/null +++ b/internal/webhook/sentinel/register.go @@ -0,0 +1,24 @@ +package sentinel + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + // handlerName is the name of the webhook handler. + handlerName = "sentinel-webhook" + // WebhookPath is the path at which the handler should be registered. + webhookPath = "/webhooks/sentinel" +) + +// RegisterWithManager registers Handler to the given manager. +func (h *Handler) RegisterWithManager(mgr manager.Manager) error { + webhook := &admission.Webhook{ + Handler: h, + RecoverPanic: true, + } + + mgr.GetWebhookServer().Register(webhookPath, webhook) + return nil +} From 1c41ceaa507241977fd4b575aa2ca7e1444c1364 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Jan 2024 17:14:57 +0530 Subject: [PATCH 080/235] added golang unit tests for utils and other misc changes --- .../operator/memberlease/memberlease_test.go | 15 +- .../operator/peerservice/peerservice_test.go | 13 +- .../poddisruptionbudget_test.go | 13 +- internal/operator/role/role_test.go | 11 +- .../operator/rolebinding/rolebinding_test.go | 11 +- .../serviceaccount/serviceaccount_test.go | 11 +- .../snapshotlease/snapshotlease_test.go | 27 ++- internal/operator/statefulset/builder.go | 11 +- internal/operator/statefulset/statefulset.go | 71 ++++-- internal/utils/image_test.go | 217 +++++++++--------- internal/utils/lease.go | 52 ++--- internal/utils/lease_test.go | 115 ++++++++++ internal/utils/store_test.go | 210 +++++++++-------- test/utils/constants.go | 7 + 14 files changed, 459 insertions(+), 325 deletions(-) create mode 100644 internal/utils/lease_test.go create mode 100644 test/utils/constants.go diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index a4a4100b1..f682b4a15 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -7,6 +7,7 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -25,9 +26,7 @@ import ( ) const ( - testEtcdName = "test-etcd" nonTargetEtcdName = "another-etcd" - testNs = "test-namespace" ) var ( @@ -37,7 +36,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string etcdReplicas int32 @@ -110,7 +109,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string etcdReplicas int32 // original replicas @@ -238,12 +237,12 @@ func TestTriggerDelete(t *testing.T) { g := NewWithT(t) t.Parallel() - nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() + nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testutils.TestNamespace).Build() nonTargetLeaseNames := []string{"another-etcd-0", "another-etcd-1"} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // *************** set up existing environment ***************** - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).WithReplicas(tc.etcdReplicas).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(tc.etcdReplicas).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllOfErr) if tc.numExistingLeases > 0 { leases, err := newMemberLeases(etcd, tc.numExistingLeases) @@ -253,7 +252,7 @@ func TestTriggerDelete(t *testing.T) { } } for _, nonTargetLeaseName := range nonTargetLeaseNames { - fakeClientBuilder.WithObjects(testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, componentName)) + fakeClientBuilder.WithObjects(testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.MemberLeaseComponentName)) } cl := fakeClientBuilder.Build() // ***************** Setup operator and test ***************** @@ -282,7 +281,7 @@ func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) cl, etcd, utils.MergeMaps[string, string](map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, }, etcd.GetDefaultLabels())) } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 94d6e46ea..e4c6bcb30 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -35,11 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" -) - var ( internalErr = errors.New("fake get internal error") apiInternalErr = apierrors.NewInternalError(internalErr) @@ -47,7 +42,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() testcases := []struct { name string svcExists bool @@ -102,7 +97,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenNoServiceExists(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string createWithPort *int32 @@ -153,7 +148,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { } func TestSyncWhenServiceExists(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string updateWithPort *int32 @@ -202,7 +197,7 @@ func TestSyncWhenServiceExists(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestPeerServiceTriggerDelete(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() deleteInternalErr := apierrors.NewInternalError(errors.New("fake delete internal error")) testCases := []struct { name string diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index de19caa68..224f1eceb 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -21,11 +21,6 @@ import ( . "github.com/onsi/gomega/gstruct" ) -const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" -) - var ( internalErr = errors.New("fake get internal error") apiInternalErr = apierrors.NewInternalError(internalErr) @@ -33,7 +28,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() testCases := []struct { name string pdbExists bool @@ -88,7 +83,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenNoPDBExists(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string etcdReplicas int32 @@ -141,7 +136,7 @@ func TestSyncWhenNoPDBExists(t *testing.T) { } func TestSyncWhenPDBExists(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string originalEtcdReplicas int32 @@ -197,7 +192,7 @@ func TestSyncWhenPDBExists(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() testCases := []struct { name string pdbExists bool diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 07cf0b0cb..b25e894aa 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -19,18 +19,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" -) - var ( internalErr = errors.New("test internal error") ) // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() getInternalErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -89,7 +84,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -137,7 +132,7 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index 32f71ab85..880890d68 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -20,18 +20,13 @@ import ( . "github.com/onsi/gomega/gstruct" ) -const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" -) - var ( internalErr = errors.New("test internal error") ) // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() getErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -90,7 +85,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string @@ -136,7 +131,7 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index cd903b93f..36972b493 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -19,11 +19,6 @@ import ( . "github.com/onsi/gomega/gstruct" ) -const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" -) - var ( internalErr = errors.New("fake get internal error") apiInternalErr = apierrors.NewInternalError(internalErr) @@ -31,7 +26,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() testCases := []struct { name string saExists bool @@ -121,7 +116,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() operator := New(cl, tc.disableAutoMount) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -170,7 +165,7 @@ func TestTriggerDelete(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) if tc.saExists { fakeClientBuilder.WithObjects(newServiceAccount(etcd, false)) diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 797956b95..98dd43583 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -7,6 +7,7 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" @@ -23,9 +24,7 @@ import ( ) const ( - testEtcdName = "test-etcd" nonTargetEtcdName = "another-etcd" - testNs = "test-namespace" ) var ( @@ -35,7 +34,7 @@ var ( // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testutils.TestNamespace) testCases := []struct { name string backupEnabled bool @@ -52,8 +51,8 @@ func TestGetExistingResourceNames(t *testing.T) { name: "successfully returns delta and full snapshot leases", backupEnabled: true, expectedLeaseNames: []string{ - fmt.Sprintf("%s-delta-snap", testEtcdName), - fmt.Sprintf("%s-full-snap", testEtcdName), + fmt.Sprintf("%s-delta-snap", testutils.TestEtcdName), + fmt.Sprintf("%s-full-snap", testutils.TestEtcdName), }, }, { @@ -95,7 +94,7 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenBackupIsEnabled(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestEtcdName).Build() testCases := []struct { name string createErr *apierrors.StatusError @@ -137,9 +136,9 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { } func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { - nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() - existingEtcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() // backup is enabled - updatedEtcd := testutils.EtcdBuilderWithoutDefaults(testEtcdName, testNs).Build() // backup is disabled + nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testutils.TestNamespace).Build() + existingEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() // backup is enabled + updatedEtcd := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() // backup is disabled testCases := []struct { name string deleteAllOfErr *apierrors.StatusError @@ -192,8 +191,8 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { - nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testNs).Build() - etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testEtcdName, testNs).WithReplicas(3) + nonTargetEtcd := testutils.EtcdBuilderWithDefaults(nonTargetEtcdName, testutils.TestNamespace).Build() + etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(3) testCases := []struct { name string backupEnabled bool @@ -275,7 +274,7 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas Name: leaseName, Namespace: etcd.Namespace, Labels: utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, }), OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, @@ -285,7 +284,7 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, }) return MatchFields(IgnoreExtras, Fields{ @@ -302,7 +301,7 @@ func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coor return doGetLatestLeases(cl, etcd, utils.MergeMaps[string, string](map[string]string{ - druidv1alpha1.LabelComponentKey: componentName, + druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, }, etcd.GetDefaultLabels())) } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 86ea470d5..e66cd9ca5 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -6,6 +6,7 @@ import ( "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" druidutils "github.com/gardener/etcd-druid/internal/utils" @@ -111,11 +112,19 @@ func (b *stsBuilder) createStatefulSetObjectMeta() { b.sts.ObjectMeta = metav1.ObjectMeta{ Name: b.etcd.Name, Namespace: b.etcd.Namespace, - Labels: b.etcd.GetDefaultLabels(), + Labels: b.getLabels(), OwnerReferences: []metav1.OwnerReference{b.etcd.GetAsOwnerReference()}, } } +func (b *stsBuilder) getLabels() map[string]string { + stsLabels := map[string]string{ + druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelAppNameKey: b.etcd.Name, + } + return utils.MergeMaps[string, string](b.etcd.GetDefaultLabels(), stsLabels) +} + func (b *stsBuilder) createStatefulSetSpec(ctx resource.OperatorContext) error { podVolumes, err := b.getPodVolumes(ctx) if err != nil { diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 477bdfb82..d1b26344d 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -4,6 +4,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" corev1 "k8s.io/api/core/v1" "k8s.io/component-base/featuregate" @@ -18,6 +19,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrGetStatefulSet druidv1alpha1.ErrorCode = "ERR_GET_STATEFUL_SET" + ErrDeleteStatefulSet druidv1alpha1.ErrorCode = "ERR_DELETE_STATEFUL_SET" + ErrSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_SYNC_STATEFUL_SET" +) + type _resource struct { client client.Client imageVector imagevector.ImageVector @@ -34,24 +41,42 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { sts, err := r.getExistingStatefulSet(ctx, etcd) - return []string{sts.Name}, err + if err != nil { + return nil, druiderr.WrapError(err, + ErrGetStatefulSet, + "GetExistingResourceNames", + fmt.Sprintf("Error getting StatefulSet: %s for etcd: %v", getObjectKey(etcd).Name, etcd.GetNamespaceName())) + } + if sts == nil { + return []string{}, nil + } + return []string{sts.Name}, nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - existingSts, err := r.getExistingStatefulSet(ctx, etcd) - if err != nil { - return fmt.Errorf("error getting existing StatefulSet: %w", err) + var ( + existingSTS *appsv1.StatefulSet + err error + ) + if existingSTS, err = r.getExistingStatefulSet(ctx, etcd); err != nil { + return druiderr.WrapError(err, + ErrSyncStatefulSet, + "Sync", + fmt.Sprintf("Error getting StatefulSet: %s for etcd: %v", getObjectKey(etcd).Name, etcd.GetNamespaceName())) + } - if existingSts == nil { - return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) + // There is no StatefulSet present. Create one. + if existingSTS == nil { + return r.createOrPatch(ctx, etcd) } - if err = r.handlePeerTLSEnabled(ctx, etcd, existingSts); err != nil { + + if err = r.handlePeerTLSEnabled(ctx, etcd, existingSTS); err != nil { return fmt.Errorf("error handling peer TLS: %w", err) } - if err = r.handleImmutableFieldUpdates(ctx, etcd, existingSts); err != nil { + if err = r.handleImmutableFieldUpdates(ctx, etcd, existingSTS); err != nil { return fmt.Errorf("error handling immutable field updates: %w", err) } - return r.createOrPatch(ctx, etcd, etcd.Spec.Replicas) + return r.createOrPatch(ctx, etcd) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { @@ -69,7 +94,9 @@ func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *dr return sts, nil } -func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { +// createOrPatchWithReplicas ensures that the StatefulSet is updated with all changes from passed in etcd but the replicas set on the StatefulSet +// are taken from the passed in replicas and not from the etcd resource. +func (r _resource) createOrPatchWithReplicas(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) mutatingFn := func() error { if builder, err := newStsBuilder(r.client, ctx.Logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { @@ -87,12 +114,17 @@ func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alph return nil } +// createOrPatch updates StatefulSet taking changes from passed in etcd resource. +func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { + return r.createOrPatchWithReplicas(ctx, etcd, etcd.Spec.Replicas) +} + func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { if existingSts.Generation > 1 && hasImmutableFieldChanged(existingSts, etcd) { ctx.Logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") if err := r.TriggerDelete(ctx, etcd); err != nil { - return fmt.Errorf("error deleting StatefulSet with immutable field updates: %v", err) + return fmt.Errorf("error deleting StatefulSet with immutable field updates: %w", err) } } return nil @@ -106,22 +138,26 @@ func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) } func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - peerTLSEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, ctx.Logger) + peerTLSEnabled, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) if err != nil { - return fmt.Errorf("error checking if peer URL TLS is enabled: %v", err) + return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) } if isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { - ctx.Logger.Info("Enabling TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) + ctx.Logger.Info("Attempting to enable TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) - if err := r.createOrPatch(ctx, etcd, *existingSts.Spec.Replicas); err != nil { - return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %v", err) + // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd resource due to + // enabling of TLS for peer communication. It preserves the current STS replicas. + if err = r.createOrPatchWithReplicas(ctx, etcd, *existingSts.Spec.Replicas); err != nil { + return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %w", err) } - tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, r.client, etcd.Namespace, etcd.Name, ctx.Logger) + tlsEnabled, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) if err != nil { return fmt.Errorf("error verifying if TLS is enabled post-patch: %v", err) } + // It usually takes time for TLS to be enabled and reflected via the lease. So first time around this will not be true. + // So instead of waiting we requeue the request to be re-tried again. if !tlsEnabled { return fmt.Errorf("failed to enable TLS for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) } @@ -129,7 +165,6 @@ func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *drui if err := deleteAllStsPods(ctx, r.client, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { return fmt.Errorf("error deleting StatefulSet pods after enabling TLS: %v", err) } - ctx.Logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) } diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index 1ce4923c1..b55b4932b 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -1,128 +1,125 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package utils import ( + "testing" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - testsample "github.com/gardener/etcd-druid/test/sample" + . "github.com/onsi/gomega" + "github.com/gardener/etcd-druid/internal/common" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/imagevector" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "k8s.io/utils/pointer" ) -var _ = Describe("Image retrieval tests", func() { +// ************************** GetEtcdImages ************************** +func TestGetEtcdImages(t *testing.T) { + tests := []struct { + name string + run func(g *WithT, etcd *druidv1alpha1.Etcd) + }{ + {"etcd spec defines etcd and etcdBR images", testWithEtcdAndEtcdBRImages}, + {"etcd spec has no image defined and image vector has etcd and etcdBR images set", testWithNoImageInSpecAndIVWithEtcdAndBRImages}, + {"", testSpecWithEtcdBRImageAndIVWithEtcdImage}, + {"", testSpecAndIVWithoutEtcdBRImage}, + {"", testWithSpecAndIVNotHavingAnyImages}, + {"", testWithNoImagesInSpecAndIVWithAllImagesWithWrapper}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() + g := NewWithT(t) + test.run(g, etcd) + }) + } +} - const ( - etcdName = "etcd-test-0" - namespace = "default" - ) - var ( - imageVector imagevector.ImageVector - etcd *druidv1alpha1.Etcd - err error - ) +func testWithEtcdAndEtcdBRImages(g *WithT, etcd *druidv1alpha1.Etcd) { + iv := createImageVector(true, true, false, false) + etcdImg, etcdBRImg, initContainerImg, err := GetEtcdImages(etcd, iv, false) + g.Expect(err).To(BeNil()) + g.Expect(etcdImg).ToNot(BeEmpty()) + g.Expect(etcdImg).To(Equal(*etcd.Spec.Etcd.Image)) + g.Expect(etcdBRImg).ToNot(BeEmpty()) + g.Expect(etcdBRImg).To(Equal(*etcd.Spec.Backup.Image)) + vectorInitContainerImage, err := iv.FindImage(common.Alpine) + g.Expect(err).To(BeNil()) + g.Expect(initContainerImg).To(Equal(vectorInitContainerImage.String())) +} - It("etcd spec defines etcd and backup-restore images", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() - imageVector = createImageVector(true, true, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - Expect(etcdImage).To(Equal(etcd.Spec.Etcd.Image)) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) +func testWithNoImageInSpecAndIVWithEtcdAndBRImages(g *WithT, etcd *druidv1alpha1.Etcd) { + etcd.Spec.Etcd.Image = nil + etcd.Spec.Backup.Image = nil + iv := createImageVector(true, true, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) + g.Expect(err).To(BeNil()) + g.Expect(etcdImage).ToNot(BeEmpty()) + vectorEtcdImage, err := iv.FindImage(common.Etcd) + g.Expect(err).To(BeNil()) + g.Expect(etcdImage).To(Equal(vectorEtcdImage.String())) + g.Expect(etcdBackupRestoreImage).ToNot(BeNil()) + vectorBackupRestoreImage, err := iv.FindImage(common.BackupRestore) + g.Expect(err).To(BeNil()) + g.Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) + vectorInitContainerImage, err := iv.FindImage(common.Alpine) + g.Expect(err).To(BeNil()) + g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) +} - It("etcd spec has no image defined and image vector has both images set", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Etcd.Image = nil - etcd.Spec.Backup.Image = nil - imageVector = createImageVector(true, true, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - vectorEtcdImage, err := imageVector.FindImage(common.Etcd) - Expect(err).To(BeNil()) - Expect(etcdImage).To(Equal(vectorEtcdImage.String())) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestore) - Expect(err).To(BeNil()) - Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) +func testSpecWithEtcdBRImageAndIVWithEtcdImage(g *WithT, etcd *druidv1alpha1.Etcd) { + etcd.Spec.Etcd.Image = nil + iv := createImageVector(true, false, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) + g.Expect(err).To(BeNil()) + g.Expect(etcdImage).ToNot(BeEmpty()) + vectorEtcdImage, err := iv.FindImage(common.Etcd) + g.Expect(err).To(BeNil()) + g.Expect(etcdImage).To(Equal(vectorEtcdImage.String())) + g.Expect(etcdBackupRestoreImage).ToNot(BeNil()) + g.Expect(etcdBackupRestoreImage).To(Equal(*etcd.Spec.Backup.Image)) + vectorInitContainerImage, err := iv.FindImage(common.Alpine) + g.Expect(err).To(BeNil()) + g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) +} - It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Etcd.Image = nil - imageVector = createImageVector(true, false, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - vectorEtcdImage, err := imageVector.FindImage(common.Etcd) - Expect(err).To(BeNil()) - Expect(etcdImage).To(Equal(vectorEtcdImage.String())) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) +func testSpecAndIVWithoutEtcdBRImage(g *WithT, etcd *druidv1alpha1.Etcd) { + etcd.Spec.Backup.Image = nil + iv := createImageVector(true, false, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) + g.Expect(err).ToNot(BeNil()) + g.Expect(etcdImage).To(BeEmpty()) + g.Expect(etcdBackupRestoreImage).To(BeEmpty()) + g.Expect(initContainerImage).To(BeEmpty()) +} - It("both spec and image vector do not have backup-restore image", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Backup.Image = nil - imageVector = createImageVector(true, false, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).ToNot(BeNil()) - Expect(etcdImage).To(BeNil()) - Expect(etcdBackupRestoreImage).To(BeNil()) - Expect(initContainerImage).To(BeNil()) - }) +func testWithSpecAndIVNotHavingAnyImages(g *WithT, etcd *druidv1alpha1.Etcd) { + etcd.Spec.Backup.Image = nil + iv := createImageVector(false, false, false, false) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) + g.Expect(err).ToNot(BeNil()) + g.Expect(etcdImage).To(BeEmpty()) + g.Expect(etcdBackupRestoreImage).To(BeEmpty()) + g.Expect(initContainerImage).To(BeEmpty()) +} - It("etcd spec has no images defined, image vector has all images, and useEtcdWrapper feature gate is turned on", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Etcd.Image = nil - etcd.Spec.Backup.Image = nil - imageVector = createImageVector(true, true, true, true) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, true) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - vectorEtcdImage, err := imageVector.FindImage(common.EtcdWrapper) - Expect(err).To(BeNil()) - Expect(etcdImage).To(Equal(vectorEtcdImage.String())) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestoreDistroless) - Expect(err).To(BeNil()) - Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) -}) +func testWithNoImagesInSpecAndIVWithAllImagesWithWrapper(g *WithT, etcd *druidv1alpha1.Etcd) { + etcd.Spec.Etcd.Image = nil + etcd.Spec.Backup.Image = nil + iv := createImageVector(true, true, true, true) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, true) + g.Expect(err).To(BeNil()) + g.Expect(etcdImage).ToNot(BeEmpty()) + vectorEtcdImage, err := iv.FindImage(common.EtcdWrapper) + g.Expect(err).To(BeNil()) + g.Expect(etcdImage).To(Equal(vectorEtcdImage.String())) + g.Expect(etcdBackupRestoreImage).ToNot(BeEmpty()) + vectorBackupRestoreImage, err := iv.FindImage(common.BackupRestoreDistroless) + g.Expect(err).To(BeNil()) + g.Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) + vectorInitContainerImage, err := iv.FindImage(common.Alpine) + g.Expect(err).To(BeNil()) + g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) +} func createImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperImage, withBackupRestoreDistrolessImage bool) imagevector.ImageVector { var imageSources []*imagevector.ImageSource diff --git a/internal/utils/lease.go b/internal/utils/lease.go index 694baf5ec..22a1649d3 100644 --- a/internal/utils/lease.go +++ b/internal/utils/lease.go @@ -18,57 +18,47 @@ import ( "context" "strconv" - "github.com/gardener/etcd-druid/pkg/common" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -// IsPeerURLTLSEnabled checks if the TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. -func IsPeerURLTLSEnabled(ctx context.Context, cli client.Client, namespace, etcdName string, logger logr.Logger) (bool, error) { - var tlsEnabledValues []bool - labels := GetMemberLeaseLabels(etcdName) +const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" + +// IsPeerURLTLSEnabledForAllMembers checks if TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. +func IsPeerURLTLSEnabledForAllMembers(ctx context.Context, cl client.Client, logger logr.Logger, namespace, etcdName string) (bool, error) { leaseList := &coordinationv1.LeaseList{} - if err := cli.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { + if err := cl.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(map[string]string{ + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelPartOfKey: etcdName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + })); err != nil { return false, err } + tlsEnabledForAllMembers := true for _, lease := range leaseList.Items { - tlsEnabled := parseAndGetTLSEnabledValue(lease, logger) - if tlsEnabled != nil { - tlsEnabledValues = append(tlsEnabledValues, *tlsEnabled) + tlsEnabled, err := parseAndGetTLSEnabledValue(lease, logger) + if err != nil { + return false, err } + tlsEnabledForAllMembers = tlsEnabledForAllMembers && tlsEnabled } - tlsEnabled := true - for _, v := range tlsEnabledValues { - tlsEnabled = tlsEnabled && v - } - return tlsEnabled, nil -} - -// PurposeMemberLease is a constant used as a purpose for etcd member lease objects. -const PurposeMemberLease = "etcd-member-lease" - -// GetMemberLeaseLabels creates a map of default labels for member lease. -func GetMemberLeaseLabels(etcdName string) map[string]string { - return map[string]string{ - common.GardenerOwnedBy: etcdName, - v1beta1constants.GardenerPurpose: PurposeMemberLease, - } + return tlsEnabledForAllMembers, nil } -func parseAndGetTLSEnabledValue(lease coordinationv1.Lease, logger logr.Logger) *bool { - const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" +func parseAndGetTLSEnabledValue(lease coordinationv1.Lease, logger logr.Logger) (bool, error) { if lease.Annotations != nil { if tlsEnabledStr, ok := lease.Annotations[peerURLTLSEnabledKey]; ok { tlsEnabled, err := strconv.ParseBool(tlsEnabledStr) if err != nil { logger.Error(err, "tls-enabled value is not a valid boolean", "namespace", lease.Namespace, "leaseName", lease.Name) - return nil + return false, err } - return &tlsEnabled + return tlsEnabled, nil } logger.V(4).Info("tls-enabled annotation not present for lease.", "namespace", lease.Namespace, "leaseName", lease.Name) } - return nil + return false, nil } diff --git a/internal/utils/lease_test.go b/internal/utils/lease_test.go new file mode 100644 index 000000000..57db730fe --- /dev/null +++ b/internal/utils/lease_test.go @@ -0,0 +1,115 @@ +package utils + +import ( + "context" + "errors" + "fmt" + "math/rand" + "testing" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIsPeerURLTLSEnabledForAllMembers(t *testing.T) { + internalErr := errors.New("fake get internal error") + apiInternalErr := apierrors.NewInternalError(internalErr) + const etcdReplicas = 3 + testCases := []struct { + name string + numETCDMembersWithTLSEnabled int + listErr *apierrors.StatusError + expectedErr *apierrors.StatusError + expectedResult bool + }{ + { + name: "should return false when none of the members have peer TLS enabled", + numETCDMembersWithTLSEnabled: 0, + expectedResult: false, + }, + { + name: "should return false when one of three do not have peer TLS enabled", + numETCDMembersWithTLSEnabled: 2, + expectedResult: false, + }, + { + name: "should return true when all members have peer TLS enabled", + numETCDMembersWithTLSEnabled: 3, + expectedResult: true, + }, + { + name: "should return error when client list call fails", + listErr: apiInternalErr, + expectedErr: apiInternalErr, + expectedResult: false, + }, + } + + g := NewWithT(t) + t.Parallel() + logger := logr.Discard() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithListError(tc.listErr) + for _, l := range createLeases(testutils.TestNamespace, testutils.TestEtcdName, etcdReplicas, tc.numETCDMembersWithTLSEnabled) { + fakeClientBuilder.WithObjects(l) + } + cl := fakeClientBuilder.Build() + tlsEnabled, err := IsPeerURLTLSEnabledForAllMembers(context.Background(), cl, logger, testutils.TestNamespace, testutils.TestEtcdName) + if tc.expectedErr != nil { + g.Expect(err).To(Equal(tc.expectedErr)) + } else { + g.Expect(tlsEnabled).To(Equal(tc.expectedResult)) + } + }) + } +} + +func createLeases(namespace, etcdName string, numLease, withTLSEnabled int) []*coordinationv1.Lease { + leases := make([]*coordinationv1.Lease, 0, numLease) + labels := map[string]string{ + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelPartOfKey: etcdName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + } + tlsEnabledCount := 0 + for i := 0; i < numLease; i++ { + var annotations map[string]string + if tlsEnabledCount < withTLSEnabled { + annotations = map[string]string{ + peerURLTLSEnabledKey: "true", + } + tlsEnabledCount++ + } else { + annotations = randomizeAnnotations() + } + lease := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", etcdName, i), + Namespace: namespace, + Annotations: annotations, + Labels: labels, + }, + } + leases = append(leases, lease) + } + return leases +} + +func randomizeAnnotations() map[string]string { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + rBool := r.Intn(2) == 1 + if rBool { + return map[string]string{ + peerURLTLSEnabledKey: "false", + } + } + return nil +} diff --git a/internal/utils/store_test.go b/internal/utils/store_test.go index 966980aa1..27527b981 100644 --- a/internal/utils/store_test.go +++ b/internal/utils/store_test.go @@ -1,118 +1,126 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package utils import ( "context" + "errors" + "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/gardener/pkg/utils/test/matchers" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - corev1 "k8s.io/api/core/v1" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ) -var _ = Describe("Store tests", func() { +func TestGetHostMountPathFromSecretRef(t *testing.T) { + const ( + hostPath = "/var/data/etcd-backup" + existingSecretName = "test-backup-secret" + nonExistingSecretName = "non-existing-backup-secret" + ) + internalErr := errors.New("test internal error") + apiInternalErr := apierrors.NewInternalError(internalErr) + logger := logr.Discard() - Describe("#GetHostMountPathFromSecretRef", func() { - var ( - ctx context.Context - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - storeSpec *druidv1alpha1.StoreSpec - sec *corev1.Secret - logger = logr.Discard() - ) - const ( - testNamespace = "test-ns" - ) + testCases := []struct { + name string + secretRefDefined bool + secretExists bool + hostPathSetInSecret bool + hostPathInSecret *string + expectedHostMountPath string + getErr *apierrors.StatusError + expectedErr *apierrors.StatusError + }{ + { + name: "no secret ref configured, should return default mount path", + secretRefDefined: false, + expectedHostMountPath: LocalProviderDefaultMountPath, + }, + { + name: "secret ref points to an unknown secret, should return an error", + secretRefDefined: true, + secretExists: false, + expectedErr: apierrors.NewNotFound(corev1.Resource("secrets"), nonExistingSecretName), + }, + { + name: "secret ref points to a secret whose data does not have path set, should return default mount path", + secretRefDefined: true, + secretExists: true, + hostPathSetInSecret: false, + expectedHostMountPath: LocalProviderDefaultMountPath, + }, + { + name: "secret ref points to a secret whose data has a path, should return the path defined in secret.Data", + secretRefDefined: true, + secretExists: true, + hostPathSetInSecret: true, + hostPathInSecret: pointer.String(hostPath), + expectedHostMountPath: hostPath, + }, + { + name: "secret exists but get fails, should return an error", + secretRefDefined: true, + secretExists: true, + getErr: apiInternalErr, + expectedErr: apiInternalErr, + }, + } - BeforeEach(func() { - ctx = context.Background() - storeSpec = &druidv1alpha1.StoreSpec{} - }) - - AfterEach(func() { - if sec != nil { - err := fakeClient.Delete(ctx, sec) - if err != nil { - Expect(err).To(matchers.BeNotFoundError()) - } + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + if tc.secretExists { + sec := createSecret(existingSecretName, testutils.TestNamespace, tc.hostPathInSecret) + fakeClientBuilder.WithObjects(sec) } - }) - - It("no secret ref configured, should return default mount path", func() { - hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(HaveOccurred()) - Expect(hostMountPath).To(Equal(LocalProviderDefaultMountPath)) - }) - - It("secret ref points to an unknown secret, should return an error", func() { - storeSpec.SecretRef = &corev1.SecretReference{ - Name: "not-to-be-found-secret-ref", - Namespace: testNamespace, + cl := fakeClientBuilder.Build() + secretName := IfConditionOr[string](tc.secretExists, existingSecretName, nonExistingSecretName) + storeSpec := createStoreSpec(tc.secretRefDefined, secretName, testutils.TestNamespace) + actualHostPath, err := GetHostMountPathFromSecretRef(context.Background(), cl, logger, storeSpec, testutils.TestNamespace) + if tc.expectedErr != nil { + g.Expect(err).ToNot(BeNil()) + var actualErr *apierrors.StatusError + g.Expect(errors.As(err, &actualErr)).To(BeTrue()) + g.Expect(actualErr.Status().Code).To(Equal(tc.expectedErr.Status().Code)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(actualHostPath).To(Equal(tc.expectedHostMountPath)) } - _, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(BeNil()) - Expect(err).To(matchers.BeNotFoundError()) }) + } +} - It("secret ref points to a secret whose data does not have path set, should return default mount path", func() { - const secretName = "backup-secret" - sec = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: testNamespace, - }, - Data: map[string][]byte{"bucketName": []byte("NDQ5YjEwZj")}, - } - Expect(fakeClient.Create(ctx, sec)).To(Succeed()) - storeSpec.SecretRef = &corev1.SecretReference{ - Name: "backup-secret", - Namespace: testNamespace, - } - hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(HaveOccurred()) - Expect(hostMountPath).To(Equal(LocalProviderDefaultMountPath)) - }) +func createStoreSpec(secretRefDefined bool, secretName, secretNamespace string) *druidv1alpha1.StoreSpec { + var secretRef *corev1.SecretReference + if secretRefDefined { + secretRef = &corev1.SecretReference{ + Name: secretName, + Namespace: secretNamespace, + } + } + return &druidv1alpha1.StoreSpec{ + SecretRef: secretRef, + } +} - It("secret ref points to a secret whose data has a path, should return the path defined in secret.Data", func() { - const ( - secretName = "backup-secret" - hostPath = "/var/data/etcd-backup" - ) - sec = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: testNamespace, - }, - Data: map[string][]byte{"bucketName": []byte("NDQ5YjEwZj"), "hostPath": []byte(hostPath)}, - } - Expect(fakeClient.Create(ctx, sec)).To(Succeed()) - storeSpec.SecretRef = &corev1.SecretReference{ - Name: "backup-secret", - Namespace: testNamespace, - } - hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(HaveOccurred()) - Expect(hostMountPath).To(Equal(hostPath)) - }) - }) -}) +func createSecret(name, namespace string, hostPath *string) *corev1.Secret { + data := map[string][]byte{ + "bucketName": []byte("NDQ5YjEwZj"), + } + if hostPath != nil { + data["hostPath"] = []byte(*hostPath) + } + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } +} diff --git a/test/utils/constants.go b/test/utils/constants.go new file mode 100644 index 000000000..db2fbfe1d --- /dev/null +++ b/test/utils/constants.go @@ -0,0 +1,7 @@ +package utils + +const ( + TestEtcdName = "test-etcd" + // TestNamespace is a test namespace to be used in tests. + TestNamespace = "test-namespace" +) From f38d57011113810a65c915a4ddbd11c70130435c Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Jan 2024 17:20:24 +0530 Subject: [PATCH 081/235] fixed unit test for utils sts --- internal/utils/statefulset_test.go | 260 +++++++++++++++++------------ test/utils/statefulset.go | 7 +- 2 files changed, 158 insertions(+), 109 deletions(-) diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index 8049aab43..48e516a39 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -1,122 +1,170 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package utils import ( "context" + "errors" + "testing" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - testsample "github.com/gardener/etcd-druid/test/sample" - "github.com/gardener/etcd-druid/test/utils" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - . "github.com/onsi/ginkgo/v2" + testutils "github.com/gardener/etcd-druid/test/utils" . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/uuid" ) -var _ = Describe("tests for statefulset utility functions", func() { - - var ( - ctx context.Context - fakeClient = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - ) +const ( + stsName = "etcd-test-0" + stsNamespace = "test-ns" +) - Describe("#IsStatefulSetReady", func() { - const ( - stsName = "etcd-test-0" - stsNamespace = "test-ns" - ) - It("statefulset has less number of ready replicas as compared to configured etcd replicas", func() { - sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 2) - sts.Generation = 1 - sts.Status.ObservedGeneration = 1 - sts.Status.Replicas = 2 - sts.Status.ReadyReplicas = 2 - ready, reasonMsg := IsStatefulSetReady(3, sts) - Expect(ready).To(BeFalse()) - Expect(reasonMsg).ToNot(BeNil()) - }) +func TestIsStatefulSetReady(t *testing.T) { + testCases := []struct { + name string + specGeneration int64 + statusObservedGeneration int64 + etcdSpecReplicas int32 + statusReadyReplicas int32 + statusCurrentReplicas int32 + statusUpdatedReplicas int32 + statusCurrentRevision string + statusUpdateRevision string + expectedStsReady bool + expectedNotReadyReasonPresent bool + }{ + { + name: "sts has less number of ready replicas as compared to configured etcd replicas", + specGeneration: 1, + statusObservedGeneration: 1, + etcdSpecReplicas: 3, + statusReadyReplicas: 2, + expectedStsReady: false, + expectedNotReadyReasonPresent: true, + }, + { + name: "sts has equal number of replicas as defined in etcd but observed generation is outdated", + specGeneration: 2, + statusObservedGeneration: 1, + etcdSpecReplicas: 3, + statusReadyReplicas: 3, + expectedStsReady: false, + expectedNotReadyReasonPresent: true, + }, + { + name: "sts has mismatching current and update revision", + specGeneration: 2, + statusObservedGeneration: 2, + etcdSpecReplicas: 3, + statusReadyReplicas: 3, + statusCurrentRevision: "etcd-main-6d5cc8f559", + statusUpdateRevision: "etcd-main-bf6b695326", + expectedStsReady: false, + expectedNotReadyReasonPresent: true, + }, + { + name: "sts has mismatching status ready and updated replicas", + specGeneration: 2, + statusObservedGeneration: 2, + etcdSpecReplicas: 3, + statusReadyReplicas: 3, + statusCurrentRevision: "etcd-main-6d5cc8f559", + statusUpdateRevision: "etcd-main-bf6b695326", + statusCurrentReplicas: 3, + statusUpdatedReplicas: 2, + expectedStsReady: false, + expectedNotReadyReasonPresent: true, + }, + { + name: "sts is completely up-to-date", + specGeneration: 2, + statusObservedGeneration: 2, + etcdSpecReplicas: 3, + statusReadyReplicas: 3, + statusCurrentRevision: "etcd-main-6d5cc8f559", + statusUpdateRevision: "etcd-main-6d5cc8f559", + statusCurrentReplicas: 3, + statusUpdatedReplicas: 3, + expectedStsReady: true, + expectedNotReadyReasonPresent: false, + }, + } - It("statefulset has equal number of replicas as defined in etcd but observed generation is outdated", func() { - sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) - sts.Generation = 2 - sts.Status.ObservedGeneration = 1 - sts.Status.Replicas = 3 - sts.Status.ReadyReplicas = 3 - ready, reasonMsg := IsStatefulSetReady(3, sts) - Expect(ready).To(BeFalse()) - Expect(reasonMsg).ToNot(BeNil()) + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 2) + sts.Generation = tc.specGeneration + sts.Status.ObservedGeneration = tc.statusObservedGeneration + sts.Status.ReadyReplicas = tc.statusReadyReplicas + sts.Status.CurrentReplicas = tc.statusCurrentReplicas + sts.Status.UpdatedReplicas = tc.statusUpdatedReplicas + sts.Status.CurrentRevision = tc.statusCurrentRevision + sts.Status.UpdateRevision = tc.statusUpdateRevision + stsReady, reasonMsg := IsStatefulSetReady(tc.etcdSpecReplicas, sts) + g.Expect(stsReady).To(Equal(tc.expectedStsReady)) + g.Expect(!IsEmptyString(reasonMsg)).To(Equal(tc.expectedNotReadyReasonPresent)) }) + } +} - It("statefulset has equal number of replicas as defined in etcd and observed generation = generation", func() { - sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) - utils.SetStatefulSetReady(sts) - ready, reasonMsg := IsStatefulSetReady(3, sts) - Expect(ready).To(BeTrue()) - Expect(len(reasonMsg)).To(BeZero()) - }) - }) +func TestGetStatefulSet(t *testing.T) { + internalErr := errors.New("test internal error") + apiInternalErr := apierrors.NewInternalError(internalErr) - Describe("#GetStatefulSet", func() { - var ( - etcd *druidv1alpha1.Etcd - stsListToCleanup *appsv1.StatefulSetList - ) - const ( - testEtcdName = "etcd-test-0" - testNamespace = "test-ns" - ) + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(3).Build() + testCases := []struct { + name string + isStsPresent bool + ownedByEtcd bool + listErr *apierrors.StatusError + expectedErr *apierrors.StatusError + }{ + { + name: "no sts found", + isStsPresent: false, + ownedByEtcd: false, + }, + { + name: "sts found but not owned by etcd", + isStsPresent: true, + ownedByEtcd: false, + }, + { + name: "sts found and owned by etcd", + isStsPresent: true, + ownedByEtcd: true, + }, + { + name: "returns error when client list fails", + isStsPresent: true, + ownedByEtcd: true, + listErr: apiInternalErr, + expectedErr: apiInternalErr, + }, + } - BeforeEach(func() { - ctx = context.TODO() - stsListToCleanup = &appsv1.StatefulSetList{} - etcd = testsample.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() - }) - - AfterEach(func() { - for _, sts := range stsListToCleanup.Items { - Expect(fakeClient.Delete(ctx, &sts)).To(Succeed()) + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithListError(tc.listErr) + if tc.isStsPresent { + etcdUID := etcd.UID + if !tc.ownedByEtcd { + etcdUID = uuid.NewUUID() + } + sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcdUID, etcd.Spec.Replicas) + fakeClientBuilder.WithObjects(sts) + } + cl := fakeClientBuilder.Build() + foundSts, err := GetStatefulSet(context.Background(), cl, etcd) + if tc.expectedErr != nil { + g.Expect(err).To(HaveOccurred()) + g.Expect(errors.Is(err, tc.expectedErr)).To(BeTrue()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + expectedStsToBeFound := tc.isStsPresent && tc.ownedByEtcd + g.Expect(foundSts != nil).To(Equal(expectedStsToBeFound)) } }) - - It("no statefulset is found irrespective of ownership", func() { - sts, err := GetStatefulSet(ctx, fakeClient, etcd) - Expect(err).To(BeNil()) - Expect(sts).To(BeNil()) - }) - - It("statefulset is present but it is not owned by etcd", func() { - sts := utils.CreateStatefulSet(etcd.Name, etcd.Namespace, uuid.NewUUID(), 3) - Expect(fakeClient.Create(ctx, sts)).To(Succeed()) - stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) - foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) - Expect(err).To(BeNil()) - Expect(foundSts).To(BeNil()) - }) - - It("found statefulset owned by etcd", func() { - sts := utils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, 3) - Expect(fakeClient.Create(ctx, sts)).To(Succeed()) - stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) - foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) - Expect(err).To(BeNil()) - Expect(foundSts).ToNot(BeNil()) - Expect(foundSts.UID).To(Equal(sts.UID)) - }) - }) -}) + } +} diff --git a/test/utils/statefulset.go b/test/utils/statefulset.go index 7b36a3507..2352f6a09 100644 --- a/test/utils/statefulset.go +++ b/test/utils/statefulset.go @@ -48,9 +48,10 @@ func CreateStatefulSet(name, namespace string, etcdUID types.UID, replicas int32 Name: name, Namespace: namespace, Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: name, + druidv1alpha1.LabelAppNameKey: name, + druidv1alpha1.LabelComponentKey: "etcd-sts", }, Annotations: nil, OwnerReferences: []metav1.OwnerReference{{ From 5fe8cd041f3e176cfa6bfd92c75fad147eccd960 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 29 Jan 2024 15:14:51 +0530 Subject: [PATCH 082/235] misc refactoring --- internal/common/constants.go | 51 +- internal/controller/compaction/config.go | 38 +- internal/controller/compaction/metrics.go | 2 +- internal/controller/compaction/reconciler.go | 525 ++++++++++++----- .../controller/compaction/reconciler_bkp.go | 283 --------- internal/controller/compaction/register.go | 2 +- .../etcdcopybackupstask/reconciler.go | 30 +- .../operator/clientservice/clientservice.go | 31 +- .../clientservice/clientservice_test.go | 63 +- internal/operator/configmap/configmap.go | 53 +- internal/operator/configmap/configmap_test.go | 324 +++++++++++ internal/operator/memberlease/memberlease.go | 4 +- .../operator/memberlease/memberlease_test.go | 21 +- internal/operator/peerservice/peerservice.go | 43 +- .../operator/peerservice/peerservice_test.go | 17 +- .../poddisruptionbudget.go | 27 +- .../poddisruptionbudget_test.go | 22 +- internal/operator/resource/types.go | 5 - internal/operator/role/role.go | 53 +- internal/operator/role/role_test.go | 22 +- internal/operator/rolebinding/rolebinding.go | 54 +- .../operator/rolebinding/rolebinding_test.go | 22 +- .../operator/serviceaccount/serviceaccount.go | 36 +- .../serviceaccount/serviceaccount_test.go | 18 +- .../operator/snapshotlease/snapshotlease.go | 16 +- .../snapshotlease/snapshotlease_test.go | 22 +- internal/operator/statefulset/builder.go | 22 +- internal/operator/statefulset/statefulset.go | 61 +- .../operator/statefulset/statefulset_test.go | 163 ++++++ internal/utils/image_test.go | 61 +- test/utils/constants.go | 21 + test/utils/errors.go | 8 + test/utils/etcd.go | 33 +- test/utils/etcdcopybackupstask.go | 37 +- test/utils/imagevector.go | 47 ++ test/utils/matcher.go | 22 +- test/utils/stsmatcher.go | 539 ++++++++++++++++++ 37 files changed, 1927 insertions(+), 871 deletions(-) delete mode 100644 internal/controller/compaction/reconciler_bkp.go create mode 100644 internal/operator/configmap/configmap_test.go create mode 100644 internal/operator/statefulset/statefulset_test.go create mode 100644 test/utils/imagevector.go diff --git a/internal/common/constants.go b/internal/common/constants.go index bdfc77d19..455fa825b 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -69,14 +69,47 @@ const ( // Constants for values to be set against druidv1alpha1.LabelComponentKey const ( - ClientServiceComponentName = "etcd-client-service" - ConfigMapComponentName = "etcd-config" - MemberLeaseComponentName = "etcd-member-lease" - SnapshotLeaseComponentName = "etcd-snapshot-lease" - PeerServiceComponentName = "etcd-peer-service" + // ClientServiceComponentName is the component name for client service resource. + ClientServiceComponentName = "etcd-client-service" + // ConfigMapComponentName is the component name for config map resource. + ConfigMapComponentName = "etcd-config" + // MemberLeaseComponentName is the component name for member lease resource. + MemberLeaseComponentName = "etcd-member-lease" + // SnapshotLeaseComponentName is the component name for snapshot lease resource. + SnapshotLeaseComponentName = "etcd-snapshot-lease" + // PeerServiceComponentName is the component name for peer service resource. + PeerServiceComponentName = "etcd-peer-service" + // PodDisruptionBudgetComponentName is the component name for pod disruption budget resource. PodDisruptionBudgetComponentName = "etcd-pdb" - RoleComponentName = "etcd-druid-role" - RoleBindingComponentName = "druid-role-binding" - ServiceAccountComponentName = "druid-service-account" - StatefulSetComponentName = "etcd-sts" + // RoleComponentName is the component name for role resource. + RoleComponentName = "etcd-druid-role" + // RoleBindingComponentName is the component name for role binding resource. + RoleBindingComponentName = "druid-role-binding" + // ServiceAccountComponentName is the component name for service account resource. + ServiceAccountComponentName = "druid-service-account" + // StatefulSetComponentName is the component name for statefulset resource. + StatefulSetComponentName = "etcd-sts" + // CompactionJobComponentName is the component name for compaction job resource. + CompactionJobComponentName = "etcd-compaction-job" + // EtcdCopyBackupTaskComponentName is the component name for copy-backup task resource. + EtcdCopyBackupTaskComponentName = "etcd-copy-backup-task" +) + +const ( + // ConfigMapCheckSumKey is the key that is set by a configmap operator and used by StatefulSet operator to + // place an annotation on the StatefulSet pods. The value contains the check-sum of the latest configmap that + // should be reflected on the pods. + ConfigMapCheckSumKey = "checksum/etcd-configmap" +) + +// Constants for container names +const ( + // EtcdContainerName is the name of the etcd container. + EtcdContainerName = "etcd" + // EtcdBackupRestoreContainerName is the name of the backup-restore container. + EtcdBackupRestoreContainerName = "backup-restore" + // ChangePermissionsInitContainerName is the name of the change permissions init container. + ChangePermissionsInitContainerName = "change-permissions" + // ChangeBackupBucketPermissionsInitContainerName is the name of the change backup bucket permissions init container. + ChangeBackupBucketPermissionsInitContainerName = "change-backup-bucket-permissions" ) diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index fe0fd70b9..af46ec88b 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -18,7 +18,7 @@ import ( "time" "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" + "github.com/gardener/etcd-druid/internal/features" flag "github.com/spf13/pflag" "k8s.io/component-base/featuregate" @@ -29,24 +29,18 @@ var featureList = []featuregate.Feature{ features.UseEtcdWrapper, } -// Flag name constants const ( - enableBackupCompactionFlagName = "enable-backup-compaction" - workersFlagName = "compaction-workers" - eventsThresholdFlagName = "etcd-events-threshold" - activeDeadlineDurationFlagName = "active-deadline-duration" - cleanupDeadlineDurationAfterCompletionFlagName = "cleanup-deadline-duration-after-completion" - metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" -) + enableBackupCompactionFlagName = "enable-backup-compaction" + workersFlagName = "compaction-workers" + eventsThresholdFlagName = "etcd-events-threshold" + activeDeadlineDurationFlagName = "active-deadline-duration" + metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" -// Default value constants -const ( - defaultEnableBackupCompaction = false - defaultCompactionWorkers = 3 - defaultEventsThreshold = 1000000 - defaultActiveDeadlineDuration = 3 * time.Hour - defaultCleanupDeadlineDurationAfterCompletion = 3 * time.Hour - defaultMetricsScrapeWaitDuration = 0 + defaultEnableBackupCompaction = false + defaultCompactionWorkers = 3 + defaultEventsThreshold = 1000000 + defaultActiveDeadlineDuration = 3 * time.Hour + defaultMetricsScrapeWaitDuration = 0 ) // Config contains configuration for the Compaction Controller. @@ -59,10 +53,6 @@ type Config struct { EventsThreshold int64 // ActiveDeadlineDuration is the duration after which a running compaction job will be killed. ActiveDeadlineDuration time.Duration - // CleanupDeadlineAfterCompletion is the duration after which an explicit cleanup of resources used by compaction will be done. - // This will ensure that resources used by compaction are eventually cleaned up. This additionally ensures that an operator - // can inspect the resources before they are cleaned-up. - CleanupDeadlineDurationAfterCompletion time.Duration // MetricsScrapeWaitDuration is the duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped MetricsScrapeWaitDuration time.Duration // FeatureGates contains the feature gates to be used by Compaction Controller. @@ -78,11 +68,9 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.Int64Var(&cfg.EventsThreshold, eventsThresholdFlagName, defaultEventsThreshold, "Total number of etcd events that can be allowed before a backup compaction job is triggered.") fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, - "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\" or similar).") - fs.DurationVar(&cfg.CleanupDeadlineDurationAfterCompletion, cleanupDeadlineDurationAfterCompletionFlagName, defaultCleanupDeadlineDurationAfterCompletion, - "Duration after which resources consumed by a completed compaction is cleaned up (Ex: \\\"300ms\\\" or \\\"2h45m\\\" or similar).\\\").") + "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\").") fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagname, defaultMetricsScrapeWaitDuration, - "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\" or similar).") + "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\").") } // Validate validates the config. diff --git a/internal/controller/compaction/metrics.go b/internal/controller/compaction/metrics.go index fa7354e08..dac70089b 100644 --- a/internal/controller/compaction/metrics.go +++ b/internal/controller/compaction/metrics.go @@ -17,7 +17,7 @@ package compaction import ( "time" - druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" + druidmetrics "github.com/gardener/etcd-druid/internal/metrics" "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/metrics" ) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 3333d3697..7f0d75983 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -16,18 +16,29 @@ package compaction import ( "context" + "fmt" + "strconv" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" + "github.com/gardener/etcd-druid/internal/features" + druidmetrics "github.com/gardener/etcd-druid/internal/metrics" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/utils/imagevector" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/component-base/featuregate" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -35,13 +46,13 @@ import ( ) const ( - // defaultETCDQuota is the default etcd quota. - defaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi + // DefaultETCDQuota is the default etcd quota. + DefaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi ) // Reconciler reconciles compaction jobs for Etcd resources. type Reconciler struct { - client client.Client + client.Client config *Config imageVector imagevector.ImageVector logger logr.Logger @@ -57,12 +68,13 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { } // NewReconcilerWithImageVector creates a new reconciler for Compaction with an ImageVector. +// This constructor will mostly be used by tests. func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { return &Reconciler{ - client: mgr.GetClient(), + Client: mgr.GetClient(), config: config, imageVector: imageVector, - logger: log.Log.WithName(controllerName), + logger: log.Log.WithName("compaction-lease-controller"), } } @@ -73,210 +85,413 @@ func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVect // Reconcile reconciles the compaction job. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.logger.Info("Compaction job reconciliation started") etcd := &druidv1alpha1.Etcd{} - if result := ctrlutils.GetLatestEtcd(ctx, r.client, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result.ReconcileResult() + if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err } - rLog := r.logger.WithValues("etcd", etcd.GetNamespaceName()) - jobKey := getJobKey(etcd) - if etcd.IsMarkedForDeletion() || !etcd.IsBackupStoreEnabled() { - return r.triggerJobDeletion(ctx, rLog, jobKey) + + if !etcd.DeletionTimestamp.IsZero() || etcd.Spec.Backup.Store == nil { + // Delete compaction job if exists + return r.delete(ctx, r.logger, etcd) } - if result := r.reconcileExistingJob(ctx, rLog, jobKey); ctrlutils.ShortCircuitReconcileFlow(result) { - return result.ReconcileResult() + logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) + + return r.reconcileJob(ctx, logger, etcd) +} + +func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { + // Update metrics for currently running compaction job, if any + job := &batchv1.Job{} + if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { + if errors.IsNotFound(err) { + logger.Info("Currently, no compaction job is running in the namespace ", etcd.Namespace) + } else { + // Error reading the object - requeue the request. + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", etcd.GetCompactionJobName(), etcd.Namespace, err) + } } - return r.reconcileJob(ctx, rLog, etcd) - /* - Get latest etcd - if it is not found then do not requeue - if there is an error then requeue with error + if job != nil && job.Name != "" { + if !job.DeletionTimestamp.IsZero() { + logger.Info("Job is already in deletion. A new job will be created only if the previous one has been deleted.", "namespace: ", job.Namespace, "name: ", job.Name) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil + } - if it's marked for deletion { - cleanup any Jobs still out there + // Check if there is one active job or not + if job.Status.Active > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) + // Don't need to requeue if the job is currently running + return ctrl.Result{}, nil } - Get Existing Job - if Job not found then continue - If error in getting Job requeue with error - If Job exists and is in deletion, requeue - If Job is active return with no-requeue - If Job succeeded record metrics and delete job and continue reconcile - If Job failed record metrics, delete job and continue reconcile - Get delta and full leases - If error requeue with error - compute difference and if difference > eventsThreshold create Job and record metrics - */ -} + // Delete job if the job succeeded + if job.Status.Succeeded > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) + if job.Status.CompletionTime != nil { + metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) + } + if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { + logger.Error(err, "Couldn't delete the successful job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while deleting successful compaction job: %v", err) + } + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() + } -func getJobKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { - return client.ObjectKey{ - Namespace: etcd.Namespace, - Name: etcd.GetCompactionJobName(), + // Delete job and requeue if the job failed + if job.Status.Failed > 0 { + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) + if job.Status.StartTime != nil { + metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) + } + err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) + if err != nil { + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while deleting failed compaction job: %v", err) + } + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil + } } -} -func (r *Reconciler) reconcileExistingJob(ctx context.Context, logger logr.Logger, jobKey client.ObjectKey) ctrlutils.ReconcileStepResult { - rjLog := logger.WithValues("operation", "reconcile-compaction-job", "job", jobKey) - job := &batchv1.Job{} - if err := r.client.Get(ctx, jobKey, job); !errors.IsNotFound(err) { - return ctrlutils.ReconcileWithError(err) + // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job + fullLease := &coordinationv1.Lease{} + + if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetFullSnapshotLeaseName()), fullLease); err != nil { + logger.Error(err, "Couldn't fetch full snap lease", "namespace", etcd.Namespace, "name", etcd.GetFullSnapshotLeaseName()) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err } - type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult - handlers := []existingJobHandler{ - r.handleJobDeletionInProgress, - r.handleActiveJob, - r.handleSuccessfulJobCompletion, - r.handleFailedJobCompletion, + deltaLease := &coordinationv1.Lease{} + if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetDeltaSnapshotLeaseName()), deltaLease); err != nil { + logger.Error(err, "Couldn't fetch delta snap lease", "namespace", etcd.Namespace, "name", etcd.GetDeltaSnapshotLeaseName()) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err } - for _, handler := range handlers { - if result := handler(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { - return result + + // Revisions have not been set yet by etcd-back-restore container. + // Skip further processing as we cannot calculate a revision delta. + if fullLease.Spec.HolderIdentity == nil || deltaLease.Spec.HolderIdentity == nil { + return ctrl.Result{}, nil + } + + full, err := strconv.ParseInt(*fullLease.Spec.HolderIdentity, 10, 64) + if err != nil { + logger.Error(err, "Can't convert holder identity of full snap lease to integer", + "namespace", fullLease.Namespace, "leaseName", fullLease.Name, "holderIdentity", fullLease.Spec.HolderIdentity) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + delta, err := strconv.ParseInt(*deltaLease.Spec.HolderIdentity, 10, 64) + if err != nil { + logger.Error(err, "Can't convert holder identity of delta snap lease to integer", + "namespace", deltaLease.Namespace, "leaseName", deltaLease.Name, "holderIdentity", deltaLease.Spec.HolderIdentity) + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, err + } + + diff := delta - full + metricNumDeltaEvents.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(float64(diff)) + + // Reconcile job only when number of accumulated revisions over the last full snapshot is more than the configured threshold value via 'events-threshold' flag + if diff >= r.config.EventsThreshold { + logger.Info("Creating etcd compaction job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) + job, err = r.createCompactionJob(ctx, logger, etcd) + if err != nil { + metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error during compaction job creation: %v", err) } + metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) } - return ctrlutils.ContinueReconcile() -} -func (r *Reconciler) getMatchingJobs(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { - // TODO: @seshachalam-yv need to fix - // r.client.List(ctx) + if job.Name != "" { + logger.Info("Current compaction job status", + "namespace", job.Namespace, "name", job.Name, "succeeded", job.Status.Succeeded) + } + + return ctrl.Result{Requeue: false}, nil } -func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { - jobObjectKey := client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetCompactionJobName()} - // Get any existing job that might exist at this time +func (r *Reconciler) delete(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { job := &batchv1.Job{} - if err := r.client.Get(ctx, jobObjectKey, job); !errors.IsNotFound(err) { - return ctrlutils.ReconcileWithError(err).ReconcileResult() + err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction job: %v", err) + } + return ctrl.Result{Requeue: false}, nil } - // TODO: @seshachalam-yv need to fix - // If there is an existing Job then check the Job status and take appropriate action. - // if !utils.IsEmptyString(job.Name) { - // if result := r.handleExistingJob(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { - // return result.ReconcileResult() - // } - // } + if job.DeletionTimestamp == nil { + logger.Info("Deleting job", "namespace", job.Namespace, "name", job.Name) + if err := client.IgnoreNotFound(r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, fmt.Errorf("error while deleting compaction job: %v", err) + } + } - // latestDeltaRevision, err := getLatestDeltaRevision(ctx, etcd.GetDeltaSnapshotLeaseName()) - return ctrlutils.ReconcileWithError(nil).ReconcileResult() + logger.Info("No compaction job is running") + return ctrl.Result{ + Requeue: false, + }, nil } -func (r *Reconciler) canScheduleJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { - r.getLatestFullSnapshotRevision(ctx, logger, etcd) -} +func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*batchv1.Job, error) { + activeDeadlineSeconds := r.config.ActiveDeadlineDuration.Seconds() -func (r *Reconciler) getLatestFullSnapshotRevision(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (int64, error) { - fullLease := &coordinationv1.Lease{} - if err := r.client.Get(ctx, client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetFullSnapshotLeaseName()}, fullLease); err != nil { - logger.Error(err, "could not fetch full snapshot lease", "lease-name", etcd.GetFullSnapshotLeaseName()) - return -1, err + _, etcdBackupImage, _, err := utils.GetEtcdImages(etcd, r.imageVector, r.config.FeatureGates[features.UseEtcdWrapper]) + if err != nil { + return nil, fmt.Errorf("couldn't fetch etcd backup image: %v", err) } - return parseRevision(fullLease) -} -func parseRevision(lease *coordinationv1.Lease) (int64, error) { - // TODO: @seshachalam-yv need to fix - // if lease.Spec.HolderIdentity == nil { + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: etcd.GetCompactionJobName(), + Namespace: etcd.Namespace, + Labels: getLabels(etcd), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: druidv1alpha1.GroupVersion.String(), + BlockOwnerDeletion: pointer.Bool(true), + Controller: pointer.Bool(true), + Kind: "Etcd", + Name: etcd.Name, + UID: etcd.UID, + }, + }, + }, + + Spec: batchv1.JobSpec{ + ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), + Completions: pointer.Int32(1), + BackoffLimit: pointer.Int32(0), + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: etcd.Spec.Annotations, + Labels: getLabels(etcd), + }, + Spec: v1.PodSpec{ + ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), + ServiceAccountName: etcd.GetServiceAccountName(), + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{{ + Name: "compact-backup", + Image: etcdBackupImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: getCompactionJobArgs(etcd, r.config.MetricsScrapeWaitDuration.String()), + }}, + }, + }, + }, + } - // } + if vms, err := getCompactionJobVolumeMounts(etcd, r.config.FeatureGates); err != nil { + return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", + etcd.Namespace, + etcd.Name, + err) + } else { + job.Spec.Template.Spec.Containers[0].VolumeMounts = vms + } - return int64(0), nil -} + if env, err := utils.GetBackupRestoreContainerEnvVars(etcd.Spec.Backup.Store); err != nil { + return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", + etcd.Namespace, + etcd.Name, + err) + } else { + job.Spec.Template.Spec.Containers[0].Env = env + } -func (r *Reconciler) handleExistingJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { - type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult - handlers := []existingJobHandler{ - r.handleJobDeletionInProgress, - r.handleActiveJob, - r.handleSuccessfulJobCompletion, - r.handleFailedJobCompletion, - } - for _, handler := range handlers { - if result := handler(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { - return result - } + if vm, err := getCompactionJobVolumes(ctx, r.Client, r.logger, etcd); err != nil { + return nil, fmt.Errorf("error creating compaction job in %v for %v : %v", + etcd.Namespace, + etcd.Name, + err) + } else { + job.Spec.Template.Spec.Volumes = vm } - return ctrlutils.ContinueReconcile() -} -func (r *Reconciler) handleJobDeletionInProgress(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { - if !job.DeletionTimestamp.IsZero() { - logger.Info("Deletion has been triggered for the job. A new job can only be created once the previously scheduled job has been deleted") - return ctrlutils.ReconcileAfter(10*time.Second, "deletion in progress, requeuing job") + if etcd.Spec.Backup.CompactionResources != nil { + job.Spec.Template.Spec.Containers[0].Resources = *etcd.Spec.Backup.CompactionResources } - return ctrlutils.ContinueReconcile() + + logger.Info("Creating job", "namespace", job.Namespace, "name", job.Name) + err = r.Create(ctx, job) + if err != nil { + return nil, err + } + + //TODO (abdasgupta): Evaluate necessity of claiming object here after creation + return job, nil } -func (r *Reconciler) handleActiveJob(_ context.Context, _ logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(1) - return ctrlutils.DoNotRequeue() +func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { + jobLabels := map[string]string{ + druidv1alpha1.LabelAppNameKey: etcd.GetCompactionJobName(), + druidv1alpha1.LabelComponentKey: common.CompactionJobComponentName, + "networking.gardener.cloud/to-dns": "allowed", + "networking.gardener.cloud/to-private-networks": "allowed", + "networking.gardener.cloud/to-public-networks": "allowed", + } + return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), jobLabels) } +func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featuregate.Feature]bool) ([]v1.VolumeMount, error) { + vms := []v1.VolumeMount{ + { + Name: "etcd-workspace-dir", + MountPath: "/var/etcd/data", + }, + } -func (r *Reconciler) handleFailedJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { - if job.Status.Failed > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) - if job.Status.StartTime != nil { - metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: job.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err != nil { + return vms, fmt.Errorf("storage provider is not recognized while fetching volume mounts") + } + switch provider { + case utils.Local: + if featureMap[features.UseEtcdWrapper] { + vms = append(vms, v1.VolumeMount{ + Name: "host-storage", + MountPath: "/home/nonroot/" + pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), + }) + } else { + vms = append(vms, v1.VolumeMount{ + Name: "host-storage", + MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), + }) } + case utils.GCS: + vms = append(vms, v1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/.gcp/", + }) + case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + vms = append(vms, v1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/var/etcd-backup/", + }) } - return ctrlutils.ContinueReconcile() + + return vms, nil } -func (r *Reconciler) handleSuccessfulJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { - if job.Status.Succeeded > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) - if job.Status.CompletionTime != nil { - metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) +func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr.Logger, etcd *druidv1alpha1.Etcd) ([]v1.Volume, error) { + vs := []v1.Volume{ + { + Name: "etcd-workspace-dir", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + + storeValues := etcd.Spec.Backup.Store + provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + if err != nil { + return vs, fmt.Errorf("could not recognize storage provider while fetching volumes") + } + switch provider { + case "Local": + hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, etcd.Namespace) + if err != nil { + return vs, fmt.Errorf("could not determine host mount path for local provider") } - if result := r.deleteJob(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { - return result + + hpt := v1.HostPathDirectory + vs = append(vs, v1.Volume{ + Name: "host-storage", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), + Type: &hpt, + }, + }, + }) + case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + if storeValues.SecretRef == nil { + return vs, fmt.Errorf("could not configure secretRef for backup store %v", provider) } - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Inc() + + vs = append(vs, v1.Volume{ + Name: "etcd-backup", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: storeValues.SecretRef.Name, + }, + }, + }) } - return ctrlutils.ContinueReconcile() + + return vs, nil } -func (r *Reconciler) triggerJobDeletion(ctx context.Context, logger logr.Logger, jobObjectKey client.ObjectKey) (ctrl.Result, error) { - dLog := logger.WithValues("operation", "delete-compaction-job", "job", jobObjectKey) - job := &batchv1.Job{} - if err := r.client.Get(ctx, jobObjectKey, job); err != nil { - if errors.IsNotFound(err) { - dLog.Info("No compaction job exists, nothing to clean up") - return ctrlutils.DoNotRequeue().ReconcileResult() - } - return ctrlutils.ReconcileWithError(err).ReconcileResult() +func getCompactionJobArgs(etcd *druidv1alpha1.Etcd, metricsScrapeWaitDuration string) []string { + command := []string{"compact"} + command = append(command, "--data-dir=/var/etcd/data/compaction.etcd") + command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp") + command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") + command = append(command, "--metrics-scrape-wait-duration="+metricsScrapeWaitDuration) + command = append(command, "--enable-snapshot-lease-renewal=true") + command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) + command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) + + var quota int64 = DefaultETCDQuota + if etcd.Spec.Etcd.Quota != nil { + quota = etcd.Spec.Etcd.Quota.Value() } + command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) - if job.DeletionTimestamp != nil { - dLog.Info("Deletion has already been triggered for the compaction job. Skipping further action.") - return ctrlutils.DoNotRequeue().ReconcileResult() + if etcd.Spec.Etcd.EtcdDefragTimeout != nil { + command = append(command, "--etcd-defrag-timeout="+etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String()) } - dLog.Info("Triggering delete of compaction job") - if result := r.deleteJob(ctx, dLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { - return result.ReconcileResult() + backupValues := etcd.Spec.Backup + if backupValues.EtcdSnapshotTimeout != nil { + command = append(command, "--etcd-snapshot-timeout="+backupValues.EtcdSnapshotTimeout.Duration.String()) } - return ctrlutils.DoNotRequeue().ReconcileResult() -} + storeValues := etcd.Spec.Backup.Store + if storeValues != nil { + provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + if err == nil { + command = append(command, "--storage-provider="+provider) + } -func (r *Reconciler) deleteJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { - if err := client.IgnoreNotFound(r.client.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { - logger.Error(err, "error when deleting compaction job") - return ctrlutils.ReconcileWithError(err) - } - return ctrlutils.ContinueReconcile() -} + if storeValues.Prefix != "" { + command = append(command, "--store-prefix="+storeValues.Prefix) + } -func (r *Reconciler) getLatestJob(ctx context.Context, objectKey client.ObjectKey) (*batchv1.Job, error) { - job := &batchv1.Job{} - if err := r.client.Get(ctx, objectKey, job); err != nil { - if errors.IsNotFound(err) { - return nil, nil + if storeValues.Container != nil { + command = append(command, "--store-container="+*(storeValues.Container)) } - return nil, err } - return job, nil + + return command } diff --git a/internal/controller/compaction/reconciler_bkp.go b/internal/controller/compaction/reconciler_bkp.go deleted file mode 100644 index 85300dcdf..000000000 --- a/internal/controller/compaction/reconciler_bkp.go +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2023 SAP SE or an SAP affiliate company -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compaction - -// -//import ( -// "context" -// "time" -// -// druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -// ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" -// druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" -// "github.com/gardener/gardener/pkg/utils/imagevector" -// "github.com/go-logr/logr" -// "github.com/prometheus/client_golang/prometheus" -// batchv1 "k8s.io/api/batch/v1" -// coordinationv1 "k8s.io/api/coordination/v1" -// "k8s.io/apimachinery/pkg/api/errors" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// ctrl "sigs.k8s.io/controller-runtime" -// "sigs.k8s.io/controller-runtime/pkg/client" -// "sigs.k8s.io/controller-runtime/pkg/log" -// "sigs.k8s.io/controller-runtime/pkg/manager" -//) -// -//const ( -// // defaultETCDQuota is the default etcd quota. -// defaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi -//) -// -//// Reconciler reconciles compaction jobs for Etcd resources. -//type Reconciler struct { -// client client.Client -// config *Config -// imageVector imagevector.ImageVector -// logger logr.Logger -//} -// -//// NewReconciler creates a new reconciler for Compaction -//func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { -// imageVector, err := ctrlutils.CreateImageVector() -// if err != nil { -// return nil, err -// } -// return NewReconcilerWithImageVector(mgr, config, imageVector), nil -//} -// -//// NewReconcilerWithImageVector creates a new reconciler for Compaction with an ImageVector. -//func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { -// return &Reconciler{ -// client: mgr.GetClient(), -// config: config, -// imageVector: imageVector, -// logger: log.Log.WithName(controllerName), -// } -//} -// -//// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch -//// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete -//// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;create;list;watch;update;patch;delete -//// +kubebuilder:rbac:groups="",resources=pods,verbs=list;watch;delete;get -// -//// Reconcile reconciles the compaction job. -//func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { -// etcd := &druidv1alpha1.Etcd{} -// if result := ctrlutils.GetLatestEtcd(ctx, r.client, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { -// return result.ReconcileResult() -// } -// rLog := r.logger.WithValues("etcd", etcd.GetNamespaceName()) -// jobKey := getJobKey(etcd) -// if etcd.IsMarkedForDeletion() || !etcd.IsBackupStoreEnabled() { -// return r.triggerJobDeletion(ctx, rLog, jobKey) -// } -// -// if result := r.reconcileExistingJob(ctx, rLog, jobKey); ctrlutils.ShortCircuitReconcileFlow(result) { -// return result.ReconcileResult() -// } -// -// return r.reconcileJob(ctx, rLog, etcd) -// /* -// Get latest etcd -// if it is not found then do not requeue -// if there is an error then requeue with error -// -// if it's marked for deletion { -// cleanup any Jobs still out there -// } -// Get Existing Job -// if Job not found then continue -// If error in getting Job requeue with error -// If Job exists and is in deletion, requeue -// If Job is active return with no-requeue -// If Job succeeded record metrics and delete job and continue reconcile -// If Job failed record metrics, delete job and continue reconcile -// Get delta and full leases -// If error requeue with error -// compute difference and if difference > eventsThreshold create Job and record metrics -// */ -// -//} -// -//func getJobKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { -// return client.ObjectKey{ -// Namespace: etcd.Namespace, -// Name: etcd.GetCompactionJobName(), -// } -//} -// -//func (r *Reconciler) reconcileExistingJob(ctx context.Context, logger logr.Logger, jobKey client.ObjectKey) ctrlutils.ReconcileStepResult { -// rjLog := logger.WithValues("operation", "reconcile-compaction-job", "job", jobKey) -// job := &batchv1.Job{} -// if err := r.client.Get(ctx, jobKey, job); !errors.IsNotFound(err) { -// return ctrlutils.ReconcileWithError(err) -// } -// -// type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult -// handlers := []existingJobHandler{ -// r.handleJobDeletionInProgress, -// r.handleActiveJob, -// r.handleSuccessfulJobCompletion, -// r.handleFailedJobCompletion, -// } -// for _, handler := range handlers { -// if result := handler(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { -// return result -// } -// } -// return ctrlutils.ContinueReconcile() -//} -// -//func (r *Reconciler) getMatchingJobs(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { -// // TODO: @seshachalam-yv need to fix -// // r.client.List(ctx) -//} -// -//func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { -// jobObjectKey := client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetCompactionJobName()} -// // Get any existing job that might exist at this time -// job := &batchv1.Job{} -// if err := r.client.Get(ctx, jobObjectKey, job); !errors.IsNotFound(err) { -// return ctrlutils.ReconcileWithError(err).ReconcileResult() -// } -// -// // TODO: @seshachalam-yv need to fix -// // If there is an existing Job then check the Job status and take appropriate action. -// // if !utils.IsEmptyString(job.Name) { -// // if result := r.handleExistingJob(ctx, rjLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { -// // return result.ReconcileResult() -// // } -// // } -// -// // latestDeltaRevision, err := getLatestDeltaRevision(ctx, etcd.GetDeltaSnapshotLeaseName()) -// return ctrlutils.ReconcileWithError(nil).ReconcileResult() -//} -// -//func (r *Reconciler) canScheduleJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) { -// r.getLatestFullSnapshotRevision(ctx, logger, etcd) -//} -// -//func (r *Reconciler) getLatestFullSnapshotRevision(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (int64, error) { -// fullLease := &coordinationv1.Lease{} -// if err := r.client.Get(ctx, client.ObjectKey{Namespace: etcd.Namespace, Name: etcd.GetFullSnapshotLeaseName()}, fullLease); err != nil { -// logger.Error(err, "could not fetch full snapshot lease", "lease-name", etcd.GetFullSnapshotLeaseName()) -// return -1, err -// } -// return parseRevision(fullLease) -//} -// -//func parseRevision(lease *coordinationv1.Lease) (int64, error) { -// // TODO: @seshachalam-yv need to fix -// // if lease.Spec.HolderIdentity == nil { -// -// // } -// -// return int64(0), nil -//} -// -//func (r *Reconciler) handleExistingJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { -// type existingJobHandler func(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult -// handlers := []existingJobHandler{ -// r.handleJobDeletionInProgress, -// r.handleActiveJob, -// r.handleSuccessfulJobCompletion, -// r.handleFailedJobCompletion, -// } -// for _, handler := range handlers { -// if result := handler(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { -// return result -// } -// } -// return ctrlutils.ContinueReconcile() -//} -// -//func (r *Reconciler) handleJobDeletionInProgress(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { -// if !job.DeletionTimestamp.IsZero() { -// logger.Info("Deletion has been triggered for the job. A new job can only be created once the previously scheduled job has been deleted") -// return ctrlutils.ReconcileAfter(10*time.Second, "deletion in progress, requeuing job") -// } -// return ctrlutils.ContinueReconcile() -//} -// -//func (r *Reconciler) handleActiveJob(_ context.Context, _ logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { -// metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(1) -// return ctrlutils.DoNotRequeue() -//} -// -//func (r *Reconciler) handleFailedJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { -// if job.Status.Failed > 0 { -// metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) -// if job.Status.StartTime != nil { -// metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: job.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) -// } -// } -// return ctrlutils.ContinueReconcile() -//} -// -//func (r *Reconciler) handleSuccessfulJobCompletion(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { -// if job.Status.Succeeded > 0 { -// metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: job.Namespace}).Set(0) -// if job.Status.CompletionTime != nil { -// metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) -// } -// if result := r.deleteJob(ctx, logger, job); ctrlutils.ShortCircuitReconcileFlow(result) { -// return result -// } -// metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: job.Namespace}).Inc() -// } -// return ctrlutils.ContinueReconcile() -//} -// -//func (r *Reconciler) triggerJobDeletion(ctx context.Context, logger logr.Logger, jobObjectKey client.ObjectKey) (ctrl.Result, error) { -// dLog := logger.WithValues("operation", "delete-compaction-job", "job", jobObjectKey) -// job := &batchv1.Job{} -// if err := r.client.Get(ctx, jobObjectKey, job); err != nil { -// if errors.IsNotFound(err) { -// dLog.Info("No compaction job exists, nothing to clean up") -// return ctrlutils.DoNotRequeue().ReconcileResult() -// } -// return ctrlutils.ReconcileWithError(err).ReconcileResult() -// } -// -// if job.DeletionTimestamp != nil { -// dLog.Info("Deletion has already been triggered for the compaction job. Skipping further action.") -// return ctrlutils.DoNotRequeue().ReconcileResult() -// } -// -// dLog.Info("Triggering delete of compaction job") -// if result := r.deleteJob(ctx, dLog, job); ctrlutils.ShortCircuitReconcileFlow(result) { -// return result.ReconcileResult() -// } -// return ctrlutils.DoNotRequeue().ReconcileResult() -//} -// -//func (r *Reconciler) deleteJob(ctx context.Context, logger logr.Logger, job *batchv1.Job) ctrlutils.ReconcileStepResult { -// if err := client.IgnoreNotFound(r.client.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { -// logger.Error(err, "error when deleting compaction job") -// return ctrlutils.ReconcileWithError(err) -// } -// return ctrlutils.ContinueReconcile() -//} -// -//func (r *Reconciler) getLatestJob(ctx context.Context, objectKey client.ObjectKey) (*batchv1.Job, error) { -// job := &batchv1.Job{} -// if err := r.client.Get(ctx, objectKey, job); err != nil { -// if errors.IsNotFound(err) { -// return nil, nil -// } -// return nil, err -// } -// return job, nil -//} diff --git a/internal/controller/compaction/register.go b/internal/controller/compaction/register.go index e9921cb32..92165c568 100644 --- a/internal/controller/compaction/register.go +++ b/internal/controller/compaction/register.go @@ -16,7 +16,7 @@ package compaction import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/internal/controller/predicate" + druidpredicates "github.com/gardener/etcd-druid/controllers/predicate" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 0439cfdb9..0a5628ec1 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -18,13 +18,12 @@ import ( "context" "fmt" "strconv" - "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ctrlutils "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/features" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/features" + "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" @@ -336,18 +335,12 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et ObjectMeta: metav1.ObjectMeta{ Name: task.GetJobName(), Namespace: task.Namespace, - Annotations: map[string]string{ - common.GardenerOwnedBy: client.ObjectKeyFromObject(task).String(), - common.GardenerOwnerType: strings.ToLower(task.Kind), - }, + Labels: getLabels(task, false), }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed, - v1beta1constants.LabelNetworkPolicyToPublicNetworks: v1beta1constants.LabelNetworkPolicyAllowed, - }, + Labels: getLabels(task, true), }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, @@ -400,6 +393,19 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et return job, nil } +func getLabels(task *druidv1alpha1.EtcdCopyBackupsTask, includeNetworkPolicyLabels bool) map[string]string { + labels := make(map[string]string) + labels[druidv1alpha1.LabelComponentKey] = common.EtcdCopyBackupTaskComponentName + labels[druidv1alpha1.LabelPartOfKey] = task.Name + labels[druidv1alpha1.LabelManagedByKey] = druidv1alpha1.LabelManagedByValue + labels[druidv1alpha1.LabelAppNameKey] = task.GetJobName() + if includeNetworkPolicyLabels { + labels[v1beta1constants.LabelNetworkPolicyToDNS] = v1beta1constants.LabelNetworkPolicyAllowed + labels[v1beta1constants.LabelNetworkPolicyToPublicNetworks] = v1beta1constants.LabelNetworkPolicyAllowed + } + return labels +} + func createJobArgs(task *druidv1alpha1.EtcdCopyBackupsTask, sourceObjStoreProvider string, targetObjStoreProvider string) []string { // Create the initial arguments for the copy-backups job. args := []string{ diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index ab311bf2f..102d44aac 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -44,33 +44,38 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * return resourceNames, druiderr.WrapError(err, ErrGetClientService, "GetExistingResourceNames", - fmt.Sprintf("Error getting client service: %s for etcd: %v", svcObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting client service: %v for etcd: %v", svcObjectKey, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, svc.Name) return resourceNames, nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - svc := emptyClientService(getObjectKey(etcd)) + objectKey := getObjectKey(etcd) + svc := emptyClientService(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { buildResource(etcd, svc) return nil }) - if err == nil { - ctx.Logger.Info("synced", "resource", "client-service", "name", svc.Name, "result", result) - return nil + if err != nil { + return druiderr.WrapError(err, + ErrSyncClientService, + "Sync", + fmt.Sprintf("Error during create or update of client service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrSyncClientService, - "Sync", - fmt.Sprintf("Error during create or update of client service for etcd: %v", etcd.GetNamespaceName()), - ) + ctx.Logger.Info("synced", "resource", "client-service", "objectKey", objectKey, "result", result) + return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of client service") - if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyClientService(objectKey))); err != nil { + ctx.Logger.Info("Triggering delete of client service", "objectKey", objectKey) + if err := r.client.Delete(ctx, emptyClientService(objectKey)); err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No Client Service found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } return druiderr.WrapError( err, ErrDeleteClientService, @@ -78,7 +83,7 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "Failed to delete client service", ) } - ctx.Logger.Info("deleted", "resource", "client-service", "name", objectKey.Name) + ctx.Logger.Info("deleted", "resource", "client-service", "objectKey", objectKey) return nil } diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 3de337f57..751cbd981 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -16,7 +16,6 @@ package clientservice import ( "context" - "errors" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -37,19 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - testEtcdName = "test-etcd" - testNs = "test-namespace" -) - -var ( - internalErr = errors.New("fake get internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() testCases := []struct { name string svcExists bool @@ -71,10 +60,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", svcExists: true, - getErr: apiInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetClientService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -95,7 +84,7 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { - g.Expect(err).To(BeNil()) + g.Expect(err).ToNot(HaveOccurred()) g.Expect(svcNames).To(Equal(tc.expectedServiceNames)) } }) @@ -105,12 +94,12 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenNoServiceExists(t *testing.T) { testCases := []struct { - name string - clientPort *int32 - backupPort *int32 - peerPort *int32 - createErr *apierrors.StatusError - expectedError *druiderr.DruidError + name string + clientPort *int32 + backupPort *int32 + peerPort *int32 + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { name: "create client service with default ports", @@ -123,10 +112,10 @@ func TestSyncWhenNoServiceExists(t *testing.T) { }, { name: "create fails when there is a create error", - createErr: apiInternalErr, - expectedError: &druiderr.DruidError{ + createErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ Code: ErrSyncClientService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -141,8 +130,8 @@ func TestSyncWhenNoServiceExists(t *testing.T) { opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) latestClientSvc, getErr := getLatestClientService(cl, etcd) - if tc.expectedError != nil { - testutils.CheckDruidError(g, tc.expectedError, err) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) } else { g.Expect(err).NotTo(HaveOccurred()) @@ -175,10 +164,10 @@ func TestSyncWhenServiceExists(t *testing.T) { { name: "update fails when there is a patch error", clientPort: pointer.Int32(2222), - patchErr: apiInternalErr, + patchErr: testutils.TestAPIInternalErr, expectedError: &druiderr.DruidError{ Code: ErrSyncClientService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -216,35 +205,35 @@ func TestTriggerDelete(t *testing.T) { testCases := []struct { name string svcExists bool - expectError *druiderr.DruidError deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError }{ { name: "no-op when client service does not exist", svcExists: false, }, { - name: "successfully delete client service", + name: "successfully delete existing client service", svcExists: true, }, { name: "returns error when client delete fails", svcExists: true, - expectError: &druiderr.DruidError{ + expectedErr: &druiderr.DruidError{ Code: ErrDeleteClientService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, - deleteErr: apiInternalErr, + deleteErr: testutils.TestAPIInternalErr, }, } g := NewWithT(t) t.Parallel() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // ********************* Setup ********************* - etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs).Build() cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() operator := New(cl) opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -256,8 +245,8 @@ func TestTriggerDelete(t *testing.T) { // ********************* Test trigger delete ********************* triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestClientService, getErr := getLatestClientService(cl, etcd) - if tc.expectError != nil { - testutils.CheckDruidError(g, tc.expectError, triggerDeleteErr) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) g.Expect(getErr).To(BeNil()) g.Expect(latestClientService).ToNot(BeNil()) } else { @@ -270,7 +259,7 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { - etcdBuilder := testutils.EtcdBuilderWithDefaults(testEtcdName, testNs) + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) if clientPort != nil { etcdBuilder.WithEtcdClientPort(clientPort) } diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 6f8f6a957..00764fde5 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -2,9 +2,11 @@ package configmap import ( "encoding/json" + "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -16,18 +18,30 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ErrGetConfigMap druidv1alpha1.ErrorCode = "ERR_GET_CONFIGMAP" + ErrDeleteConfigMap druidv1alpha1.ErrorCode = "ERR_DELETE_CONFIGMAP" + ErrSyncConfigMap druidv1alpha1.ErrorCode = "ERR_SYNC_CONFIGMAP" +) + +const etcdConfigKey = "etcd.conf.yaml" + type _resource struct { client client.Client } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) + objKey := getObjectKey(etcd) cm := &corev1.ConfigMap{} - if err := r.client.Get(ctx, getObjectKey(etcd), cm); err != nil { + if err := r.client.Get(ctx, objKey, cm); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } - return resourceNames, err + return nil, druiderr.WrapError(err, + ErrGetConfigMap, + "GetExistingResourceNames", + fmt.Sprintf("Error getting ConfigMap: %v for etcd: %v", objKey, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, cm.Name) return resourceNames, nil @@ -35,24 +49,44 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { cm := emptyConfigMap(getObjectKey(etcd)) - _, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { + result, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { return buildResource(etcd, cm) }) if err != nil { - return err + return druiderr.WrapError(err, + ErrSyncConfigMap, + "Sync", + fmt.Sprintf("Error during create or update of configmap for etcd: %v", etcd.GetNamespaceName())) } checkSum, err := computeCheckSum(cm) if err != nil { - return err + return druiderr.WrapError(err, + ErrSyncConfigMap, + "Sync", + fmt.Sprintf("Error when computing CheckSum for configmap for etcd: %v", etcd.GetNamespaceName())) } - ctx.Data[resource.ConfigMapCheckSumKey] = checkSum + ctx.Data[common.ConfigMapCheckSumKey] = checkSum + ctx.Logger.Info("synced", "resource", "configmap", "name", cm.Name, "result", result) return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) - return client.IgnoreNotFound(r.client.Delete(ctx, emptyConfigMap(objectKey))) + if err := r.client.Delete(ctx, emptyConfigMap(objectKey)); err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No ConfigMap found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } + return druiderr.WrapError( + err, + ErrDeleteConfigMap, + "TriggerDelete", + "Failed to delete configmap", + ) + } + ctx.Logger.Info("deleted", "resource", "configmap", "objectKey", objectKey) + return nil } func New(client client.Client) resource.Operator { @@ -71,7 +105,8 @@ func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { cm.Namespace = etcd.Namespace cm.Labels = getLabels(etcd) cm.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} - cm.Data = map[string]string{"etcd.conf.yaml": string(cfgYaml)} + cm.Data = map[string]string{etcdConfigKey: string(cfgYaml)} + return nil } @@ -102,7 +137,7 @@ func emptyConfigMap(objectKey client.ObjectKey) *corev1.ConfigMap { func computeCheckSum(cm *corev1.ConfigMap) (string, error) { jsonData, err := json.Marshal(cm.Data) if err != nil { - return "", nil + return "", err } return gardenerutils.ComputeSHA256Hex(jsonData), nil } diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go new file mode 100644 index 000000000..779aec717 --- /dev/null +++ b/internal/operator/configmap/configmap_test.go @@ -0,0 +1,324 @@ +package configmap + +import ( + "context" + "fmt" + "strconv" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() + testCases := []struct { + name string + cmExists bool + getErr *apierrors.StatusError + expectedErr *druiderr.DruidError + expectedConfigMapNames []string + }{ + { + name: "should return empty slice when no configmap found", + cmExists: false, + expectedConfigMapNames: []string{}, + }, + { + name: "should return the existing congigmap name", + cmExists: true, + expectedConfigMapNames: []string{etcd.GetConfigMapName()}, + }, + { + name: "should return error when get client get fails", + cmExists: true, + getErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetConfigMap, + Cause: testutils.TestAPIInternalErr, + Operation: "GetExistingResourceNames", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + if tc.cmExists { + fakeClientBuilder.WithObjects(newConfigMap(g, etcd)) + } + operator := New(fakeClientBuilder.Build()) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + cmNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cmNames).To(Equal(tc.expectedConfigMapNames)) + } + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSyncWhenNoConfigMapExists(t *testing.T) { + testCases := []struct { + name string + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "should create when no configmap exists", + }, + { + name: "return error when create client request fails", + createErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncConfigMap, + Cause: testutils.TestAPIInternalErr, + Operation: "Sync", + }, + }, + } + g := NewWithT(t) + t.Parallel() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().WithPeerTLS().Build() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, etcd) + latestConfigMap, getErr := getLatestConfigMap(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(latestConfigMap).ToNot(BeNil()) + matchConfigMap(g, etcd, *latestConfigMap) + } + }) + } +} + +func TestSyncWhenConfigMapExists(t *testing.T) { + testCases := []struct { + name string + patchErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "update configmap when peer TLS communication is enabled", + }, + { + name: "returns error when patch client request fails", + patchErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncConfigMap, + Cause: testutils.TestAPIInternalErr, + Operation: "Sync", + }, + }, + } + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + originalEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().Build() + cl := testutils.NewFakeClientBuilder(). + WithPatchError(tc.patchErr). + WithObjects(newConfigMap(g, originalEtcd)). + Build() + updatedEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().WithPeerTLS().Build() + updatedEtcd.UID = originalEtcd.UID + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + err := operator.Sync(opCtx, updatedEtcd) + latestConfigMap, getErr := getLatestConfigMap(cl, updatedEtcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + g.Expect(getErr).NotTo(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(latestConfigMap).ToNot(BeNil()) + matchConfigMap(g, updatedEtcd, *latestConfigMap) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +func TestTriggerDelete(t *testing.T) { + testCases := []struct { + name string + cmExists bool + deleteErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "no-op when configmap does not exist", + cmExists: false, + }, + { + name: "successfully delete existing configmap", + cmExists: true, + }, + { + name: "return error when client delete fails", + cmExists: true, + deleteErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrDeleteConfigMap, + Cause: testutils.TestAPIInternalErr, + Operation: "TriggerDelete", + }, + }, + } + g := NewWithT(t) + t.Parallel() + + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // ********************* Setup ********************* + cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() + operator := New(cl) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + if tc.cmExists { + syncErr := operator.Sync(opCtx, etcd) + g.Expect(syncErr).ToNot(HaveOccurred()) + ensureConfigMapExists(g, cl, etcd) + } + // ********************* Test trigger delete ********************* + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + latestConfigMap, getErr := getLatestConfigMap(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) + g.Expect(getErr).To(BeNil()) + g.Expect(latestConfigMap).ToNot(BeNil()) + } else { + g.Expect(triggerDeleteErr).NotTo(HaveOccurred()) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } + }) + } +} + +// ---------------------------- Helper Functions ----------------------------- +func newConfigMap(g *WithT, etcd *druidv1alpha1.Etcd) *corev1.ConfigMap { + cm := emptyConfigMap(getObjectKey(etcd)) + err := buildResource(etcd, cm) + g.Expect(err).ToNot(HaveOccurred()) + return cm +} + +func ensureConfigMapExists(g *WithT, cl client.WithWatch, etcd *druidv1alpha1.Etcd) { + cm, err := getLatestConfigMap(cl, etcd) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cm).ToNot(BeNil()) +} + +func getLatestConfigMap(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.ConfigMap, error) { + cm := &corev1.ConfigMap{} + err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetConfigMapName(), Namespace: etcd.Namespace}, cm) + return cm, err +} + +func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.ConfigMap) { + expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + druidv1alpha1.LabelComponentKey: common.ConfigMapComponentName, + druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), + }) + g.Expect(actualConfigMap).To(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Name": Equal(etcd.GetConfigMapName()), + "Namespace": Equal(etcd.Namespace), + "Labels": testutils.MatchResourceLabels(expectedLabels), + "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), + }), + "Spec": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Data": Not(BeNil()), + }), + })) + // Validate the etcd config data + actualETCDConfigYAML := actualConfigMap.Data[etcdConfigKey] + actualETCDConfig := make(map[string]interface{}) + err := yaml.Unmarshal([]byte(actualETCDConfigYAML), &actualETCDConfig) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ + "name": Equal(fmt.Sprintf("etcd-%s", etcd.UID[:6])), + "data-dir": Equal("/var/etcd/data/new.etcd"), + "metrics": Equal(string(druidv1alpha1.Basic)), + "snapshot-count": Equal(int64(75000)), + "enable-v2": Equal(false), + "quota-backend-bytes": Equal(etcd.Spec.Etcd.Quota.Value()), + "initial-cluster-token": Equal("etcd-cluster"), + "initial-cluster-state": Equal("new"), + "auto-compaction-mode": Equal(string(utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic))), + "auto-compaction-retention": Equal(utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention)), + })) + matchClientTLSRelatedConfiguration(g, etcd, actualETCDConfig) + matchPeerTLSRelatedConfiguration(g, etcd, actualETCDConfig) +} + +func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { + if etcd.Spec.Etcd.ClientUrlTLS != nil { + g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "client-transport-security": MatchKeys(IgnoreExtras, Keys{ + "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), + "client-cert-auth": Equal(true), + "trusted-ca-file": Equal("/var/etcd/ssl/client/ca/ca.crt"), + "auto-tls": Equal(false), + }), + })) + } else { + g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ + "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + })) + g.Expect(actualETCDConfig).ToNot(HaveKey("client-transport-security")) + } +} + +func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { + if etcd.Spec.Etcd.PeerUrlTLS != nil { + g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ + "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ + "cert-file": Equal("/var/etcd/ssl/peer/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/peer/server/tls.key"), + "client-cert-auth": Equal(true), + "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), + "auto-tls": Equal(false), + }), + "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))))), + })) + } else { + g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ + "advertise-client-urls": Equal(fmt.Sprintf("http@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))))), + })) + g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) + } +} diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 8b8737ee3..51cd690e3 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -77,9 +77,9 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a return druiderr.WrapError(err, ErrSyncMemberLease, "Sync", - fmt.Sprintf("Error syncing member lease: %s for etcd: %v", objKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error syncing member lease: %v for etcd: %v", objKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("triggered create or update of member lease", "lease", objKey, "operationResult", opResult) + ctx.Logger.Info("triggered create or update of member lease", "objectKey", objKey, "operationResult", opResult) return nil } diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index f682b4a15..6aed79008 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -29,11 +29,6 @@ const ( nonTargetEtcdName = "another-etcd" ) -var ( - internalErr = errors.New("test internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace) @@ -68,10 +63,10 @@ func TestGetExistingResourceNames(t *testing.T) { }, { name: "should return error when list fails", - listErr: apiInternalErr, + listErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrListMemberLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -134,10 +129,10 @@ func TestSync(t *testing.T) { name: "should return error when client create fails", etcdReplicas: 3, numExistingLeases: 0, - createErr: apiInternalErr, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncMemberLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -145,10 +140,10 @@ func TestSync(t *testing.T) { name: "should return error when client get fails", etcdReplicas: 3, numExistingLeases: 0, - getErr: apiInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncMemberLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -226,10 +221,10 @@ func TestTriggerDelete(t *testing.T) { name: "returns error when client delete fails", etcdReplicas: 3, numExistingLeases: 3, - deleteAllOfErr: apiInternalErr, + deleteAllOfErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrDeleteMemberLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, }, diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 6ed38c047..a1a3417be 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -46,36 +46,41 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - svc := emptyPeerService(getObjectKey(etcd)) + objectKey := getObjectKey(etcd) + svc := emptyPeerService(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { buildResource(etcd, svc) return nil }) - if err == nil { - ctx.Logger.Info("synced", "resource", "peer-service", "name", svc.Name, "result", result) - return nil + if err != nil { + return druiderr.WrapError(err, + ErrSyncPeerService, + "Sync", + fmt.Sprintf("Error during create or update of peer service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrSyncPeerService, - "Sync", - fmt.Sprintf("Error during create or update of peer service: %s for etcd: %v", svc.Name, etcd.GetNamespaceName()), - ) + ctx.Logger.Info("synced", "resource", "peer-service", "objectKey", objectKey, "result", result) + return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of peer service") - err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPeerService(objectKey))) - if err == nil { - ctx.Logger.Info("deleted", "resource", "peer-service", "name", objectKey.Name) - return nil + err := r.client.Delete(ctx, emptyPeerService(objectKey)) + if err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No Peer Service found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } + return druiderr.WrapError( + err, + ErrDeletePeerService, + "TriggerDelete", + fmt.Sprintf("Failed to delete peer service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError( - err, - ErrDeletePeerService, - "TriggerDelete", - fmt.Sprintf("Failed to delete peer service: %s for etcd: %v", objectKey.Name, etcd.GetNamespaceName()), - ) + ctx.Logger.Info("deleted", "resource", "peer-service", "objectKey", objectKey) + return nil } func New(client client.Client) resource.Operator { diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index e4c6bcb30..ead7378c4 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -35,11 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - internalErr = errors.New("fake get internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() @@ -64,10 +59,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", svcExists: true, - getErr: apiInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetPeerService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -113,10 +108,10 @@ func TestSyncWhenNoServiceExists(t *testing.T) { }, { name: "returns error when client create fails", - createErr: apiInternalErr, + createErr: testutils.TestAPIInternalErr, expectedError: &druiderr.DruidError{ Code: ErrSyncPeerService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -162,10 +157,10 @@ func TestSyncWhenServiceExists(t *testing.T) { { name: "update fails when there is a patch error", updateWithPort: pointer.Int32(2222), - patchErr: apiInternalErr, + patchErr: testutils.TestAPIInternalErr, expectedError: &druiderr.DruidError{ Code: ErrSyncPeerService, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index fc1c5e5eb..8fbd92101 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -28,34 +28,37 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) + objectKey := getObjectKey(etcd) pdb := &policyv1.PodDisruptionBudget{} - if err := r.client.Get(ctx, getObjectKey(etcd), pdb); err != nil { + if err := r.client.Get(ctx, objectKey, pdb); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } return resourceNames, druiderr.WrapError(err, ErrGetPodDisruptionBudget, "GetExistingResourceNames", - fmt.Sprintf("Error getting PDB: %s for etcd: %v", pdb.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, pdb.Name) return resourceNames, nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - pdb := emptyPodDisruptionBudget(getObjectKey(etcd)) + objectKey := getObjectKey(etcd) + pdb := emptyPodDisruptionBudget(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { buildResource(etcd, pdb) return nil }) - if err == nil { - ctx.Logger.Info("synced", "resource", "pod-disruption-budget", "name", pdb.Name, "result", result) - return nil + if err != nil { + return druiderr.WrapError(err, + ErrSyncPodDisruptionBudget, + "Sync", + fmt.Sprintf("Error during create or update of PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrSyncPodDisruptionBudget, - "Sync", - fmt.Sprintf("Error during create or update of PDB: %s for etcd: %v", pdb.Name, etcd.GetNamespaceName())) + ctx.Logger.Info("synced", "resource", "pod-disruption-budget", "objectKey", objectKey, "result", result) + return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { @@ -65,9 +68,9 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph return druiderr.WrapError(err, ErrDeletePodDisruptionBudget, "TriggerDelete", - fmt.Sprintf("Failed to delete PDB: %s for etcd: %v", pdbObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete PDB: %v for etcd: %v", pdbObjectKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("deleted", "resource", "pod-disruption-budget", "name", pdbObjectKey.Name) + ctx.Logger.Info("deleted", "resource", "pod-disruption-budget", "objectKey", pdbObjectKey) return nil } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 224f1eceb..f5946b072 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -2,7 +2,6 @@ package poddistruptionbudget import ( "context" - "errors" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -21,11 +20,6 @@ import ( . "github.com/onsi/gomega/gstruct" ) -var ( - internalErr = errors.New("fake get internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() @@ -50,10 +44,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", pdbExists: true, - getErr: apiInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetPodDisruptionBudget, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -103,10 +97,10 @@ func TestSyncWhenNoPDBExists(t *testing.T) { }, { name: "returns error when client create fails", - createErr: apiInternalErr, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncPodDisruptionBudget, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -156,10 +150,10 @@ func TestSyncWhenPDBExists(t *testing.T) { originalEtcdReplicas: 1, updatedEtcdReplicas: 3, expectedPDBMinAvailable: 0, - patchErr: apiInternalErr, + patchErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncPodDisruptionBudget, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -210,10 +204,10 @@ func TestTriggerDelete(t *testing.T) { { name: "returns error when client delete fails", pdbExists: true, - deleteErr: apiInternalErr, + deleteErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrDeletePodDisruptionBudget, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, }, diff --git a/internal/operator/resource/types.go b/internal/operator/resource/types.go index bad88745d..3f1dfbe98 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/resource/types.go @@ -7,11 +7,6 @@ import ( "github.com/go-logr/logr" ) -const ( - // ConfigMapCheckSumKey is the key which stores the latest checksum value of the ConfigMap changes as part of an etcd spec reconciliation. - ConfigMapCheckSumKey = "checksum/etcd-configmap" -) - // OperatorContext holds the underline context.Context along with additional data that needs to be passed from one reconcile-step to another in a multistep reconciliation run. type OperatorContext struct { context.Context diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 4718a8e17..3ab02ce61 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -27,48 +27,55 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) - roleObjectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd) role := &rbacv1.Role{} - if err := r.client.Get(ctx, roleObjectKey, role); err != nil { + if err := r.client.Get(ctx, objectKey, role); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } return resourceNames, druiderr.WrapError(err, ErrGetRole, "GetExistingResourceNames", - fmt.Sprintf("Error getting role: %s for etcd: %v", roleObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting role: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, role.Name) return resourceNames, nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - role := emptyRole(etcd) + objectKey := getObjectKey(etcd) + role := emptyRole(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { buildResource(etcd, role) return nil }) - if err == nil { - ctx.Logger.Info("synced", "resource", "role", "name", role.Name, "result", result) + if err != nil { + return druiderr.WrapError(err, + ErrSyncRole, + "Sync", + fmt.Sprintf("Error during create or update of role %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrSyncRole, - "Sync", - fmt.Sprintf("Error during create or update of role %s for etcd: %v", etcd.GetRoleName(), etcd.GetNamespaceName()), - ) + ctx.Logger.Info("synced", "resource", "role", "objectKey", objectKey, "result", result) + return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of role") - err := client.IgnoreNotFound(r.client.Delete(ctx, emptyRole(etcd))) - if err == nil { - ctx.Logger.Info("deleted", "resource", "role", "name", etcd.GetRoleName()) + objectKey := getObjectKey(etcd) + ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) + if err := r.client.Delete(ctx, emptyRole(objectKey)); err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No Role found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } + return druiderr.WrapError(err, + ErrDeleteRole, + "TriggerDelete", + fmt.Sprintf("Failed to delete role: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrDeleteRole, - "TriggerDelete", - fmt.Sprintf("Failed to delete role: %s for etcd: %v", etcd.GetRoleName(), etcd.GetNamespaceName()), - ) + ctx.Logger.Info("deleted", "resource", "role", "objectKey", objectKey) + return nil } func New(client client.Client) resource.Operator { @@ -81,11 +88,11 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace} } -func emptyRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { +func emptyRole(objectKey client.ObjectKey) *rbacv1.Role { return &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetRoleName(), - Namespace: etcd.Namespace, + Name: objectKey.Name, + Namespace: objectKey.Namespace, }, } } diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index b25e894aa..15236725e 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -2,7 +2,6 @@ package role import ( "context" - "errors" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -19,14 +18,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - internalErr = errors.New("test internal error") -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - getInternalErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string roleExists bool @@ -48,10 +42,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", roleExists: true, - getErr: getInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetRole, - Cause: getInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -85,7 +79,6 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string createErr *apierrors.StatusError @@ -96,10 +89,10 @@ func TestSync(t *testing.T) { }, { name: "create role fails when client create fails", - createErr: internalStatusErr, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncRole, - Cause: internalStatusErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -133,7 +126,6 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string roleExists bool @@ -147,10 +139,10 @@ func TestTriggerDelete(t *testing.T) { { name: "delete fails due to failing client delete", roleExists: true, - deleteErr: internalStatusErr, + deleteErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrDeleteRole, - Cause: internalStatusErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, }, @@ -192,7 +184,7 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { - role := emptyRole(etcd) + role := emptyRole(getObjectKey(etcd)) buildResource(etcd, role) return role } diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index e092f2daa..fcdc4ce8a 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -27,48 +27,56 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) - rbObjectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd) rb := &rbacv1.RoleBinding{} - if err := r.client.Get(ctx, rbObjectKey, rb); err != nil { + if err := r.client.Get(ctx, objectKey, rb); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } return resourceNames, druiderr.WrapError(err, ErrGetRoleBinding, "GetExistingResourceNames", - fmt.Sprintf("Error getting role-binding: %s for etcd: %v", rbObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, rb.Name) return resourceNames, nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of role") - err := client.IgnoreNotFound(r.client.Delete(ctx, emptyRoleBinding(etcd))) - if err == nil { - ctx.Logger.Info("deleted", "resource", "role-binding", "name", etcd.GetRoleBindingName()) + objectKey := getObjectKey(etcd) + ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) + err := r.client.Delete(ctx, emptyRoleBinding(objectKey)) + if err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No RoleBinding found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } + return druiderr.WrapError(err, + ErrDeleteRoleBinding, + "TriggerDelete", + fmt.Sprintf("Failed to delete role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrDeleteRoleBinding, - "TriggerDelete", - fmt.Sprintf("Failed to delete role-binding: %s for etcd: %v", etcd.GetRoleBindingName(), etcd.GetNamespaceName()), - ) + ctx.Logger.Info("deleted", "resource", "role-binding", "objectKey", objectKey) + return nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - rb := emptyRoleBinding(etcd) + objectKey := getObjectKey(etcd) + rb := emptyRoleBinding(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { buildResource(etcd, rb) return nil }) - if err == nil { - ctx.Logger.Info("synced", "resource", "role", "name", rb.Name, "result", result) + if err != nil { + return druiderr.WrapError(err, + ErrSyncRoleBinding, + "Sync", + fmt.Sprintf("Error during create or update of role-binding %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrSyncRoleBinding, - "Sync", - fmt.Sprintf("Error during create or update of role-binding %s for etcd: %v", etcd.GetRoleBindingName(), etcd.GetNamespaceName()), - ) + ctx.Logger.Info("synced", "resource", "role", "objectKey", objectKey, "result", result) + return nil } func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { @@ -92,11 +100,11 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace} } -func emptyRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { +func emptyRoleBinding(objKey client.ObjectKey) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetRoleBindingName(), - Namespace: etcd.Namespace, + Name: objKey.Name, + Namespace: objKey.Namespace, }, } } diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index 880890d68..2fd1e35d9 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -2,7 +2,6 @@ package rolebinding import ( "context" - "errors" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -20,14 +19,9 @@ import ( . "github.com/onsi/gomega/gstruct" ) -var ( - internalErr = errors.New("test internal error") -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - getErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string roleBindingExists bool @@ -49,10 +43,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return error when get fails", roleBindingExists: true, - getErr: getErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetRoleBinding, - Cause: getErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -86,7 +80,6 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSync(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string createErr *apierrors.StatusError @@ -97,10 +90,10 @@ func TestSync(t *testing.T) { }, { name: "create role fails when client create fails", - createErr: internalStatusErr, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncRoleBinding, - Cause: internalStatusErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -132,7 +125,6 @@ func TestSync(t *testing.T) { // ----------------------------- TriggerDelete ------------------------------- func TestTriggerDelete(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - internalStatusErr := apierrors.NewInternalError(internalErr) testCases := []struct { name string roleBindingExists bool @@ -146,10 +138,10 @@ func TestTriggerDelete(t *testing.T) { { name: "delete fails due to failing client delete", roleBindingExists: true, - deleteErr: internalStatusErr, + deleteErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrDeleteRoleBinding, - Cause: internalStatusErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, }, @@ -191,7 +183,7 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { - rb := emptyRoleBinding(etcd) + rb := emptyRoleBinding(getObjectKey(etcd)) buildResource(etcd, rb) return rb } diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 01235b546..a6db8ea0f 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -30,46 +30,52 @@ type _resource struct { func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) sa := &corev1.ServiceAccount{} - saObjectKey := getObjectKey(etcd) - if err := r.client.Get(ctx, saObjectKey, sa); err != nil { + objectKey := getObjectKey(etcd) + if err := r.client.Get(ctx, objectKey, sa); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } return resourceNames, druiderr.WrapError(err, ErrGetServiceAccount, "GetExistingResourceNames", - fmt.Sprintf("Error getting service account: %s for etcd: %v", saObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } resourceNames = append(resourceNames, sa.Name) return resourceNames, nil } func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - sa := emptyServiceAccount(getObjectKey(etcd)) + objectKey := getObjectKey(etcd) + sa := emptyServiceAccount(objectKey) opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, sa, func() error { buildResource(etcd, sa, !r.disableAutoMount) return nil }) - if err == nil { - ctx.Logger.Info("synced", "resource", "service-account", "name", sa.Name, "result", opResult) - return nil + if err != nil { + return druiderr.WrapError(err, + ErrSyncServiceAccount, + "Sync", + fmt.Sprintf("Error during create or update of service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } - return druiderr.WrapError(err, - ErrSyncServiceAccount, - "Sync", - fmt.Sprintf("Error during create or update of service account: %s for etcd: %v", sa.Name, etcd.GetNamespaceName())) + ctx.Logger.Info("synced", "resource", "service-account", "objectKey", objectKey, "result", opResult) + return nil } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of service account") - saObjectKey := getObjectKey(etcd) - if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyServiceAccount(saObjectKey))); err != nil { + objectKey := getObjectKey(etcd) + if err := r.client.Delete(ctx, emptyServiceAccount(objectKey)); err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No ServiceAccount found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } return druiderr.WrapError(err, ErrDeleteServiceAccount, "TriggerDelete", - fmt.Sprintf("Failed to delete service account: %s for etcd: %v", saObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("deleted", "resource", "service-account", "name", saObjectKey.Name) + ctx.Logger.Info("deleted", "resource", "service-account", "objectKey", objectKey) return nil } diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index 36972b493..e889ebfa3 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -2,7 +2,6 @@ package serviceaccount import ( "context" - "errors" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -19,11 +18,6 @@ import ( . "github.com/onsi/gomega/gstruct" ) -var ( - internalErr = errors.New("fake get internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() @@ -48,10 +42,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return err when client get fails", saExists: true, - getErr: apiInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetServiceAccount, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -102,10 +96,10 @@ func TestSync(t *testing.T) { }, { name: "should return err when client create fails", - createErr: apiInternalErr, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncServiceAccount, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -152,10 +146,10 @@ func TestTriggerDelete(t *testing.T) { { name: "returns error when client delete fails", saExists: true, - deleteErr: apiInternalErr, + deleteErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrDeleteServiceAccount, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, }, diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 2a262dfc5..050ed0326 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -33,27 +33,27 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * // because currently snapshot lease do not have proper labels on them. In this new code // we will add the labels. // TODO: Once all snapshot leases have a purpose label on them, then we can use List instead of individual Get calls. - deltaSnapshotLease, err := r.getLease(ctx, - client.ObjectKey{Name: etcd.GetDeltaSnapshotLeaseName(), Namespace: etcd.Namespace}) + deltaSnapshotObjectKey := client.ObjectKey{Name: etcd.GetDeltaSnapshotLeaseName(), Namespace: etcd.Namespace} + deltaSnapshotLease, err := r.getLease(ctx, deltaSnapshotObjectKey) if err != nil { return resourceNames, &druiderr.DruidError{ Code: ErrGetSnapshotLease, Cause: err, Operation: "GetExistingResourceNames", - Message: fmt.Sprintf("Error getting delta snapshot lease: %s for etcd: %v", etcd.GetDeltaSnapshotLeaseName(), etcd.GetNamespaceName()), + Message: fmt.Sprintf("Error getting delta snapshot lease: %v for etcd: %v", deltaSnapshotObjectKey, etcd.GetNamespaceName()), } } if deltaSnapshotLease != nil { resourceNames = append(resourceNames, deltaSnapshotLease.Name) } - fullSnapshotLease, err := r.getLease(ctx, - client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace}) + fullSnapshotObjectKey := client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace} + fullSnapshotLease, err := r.getLease(ctx, fullSnapshotObjectKey) if err != nil { return resourceNames, &druiderr.DruidError{ Code: ErrGetSnapshotLease, Cause: err, Operation: "GetExistingResourceNames", - Message: fmt.Sprintf("Error getting full snapshot lease: %s for etcd: %v", etcd.GetFullSnapshotLeaseName(), etcd.GetNamespaceName()), + Message: fmt.Sprintf("Error getting full snapshot lease: %v for etcd: %v", fullSnapshotObjectKey, etcd.GetNamespaceName()), } } if fullSnapshotLease != nil { @@ -142,9 +142,9 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a return druiderr.WrapError(err, ErrSyncSnapshotLease, "Sync", - fmt.Sprintf("Error syncing snapshot lease: %s for etcd: %v", leaseObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error syncing snapshot lease: %v for etcd: %v", leaseObjectKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("triggered create or update of snapshot lease", "lease", leaseObjectKey, "operationResult", opResult) + ctx.Logger.Info("triggered create or update of snapshot lease", "objectKey", leaseObjectKey, "operationResult", opResult) return nil } diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 98dd43583..67f784699 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -2,7 +2,6 @@ package snapshotlease import ( "context" - "errors" "fmt" "testing" @@ -27,11 +26,6 @@ const ( nonTargetEtcdName = "another-etcd" ) -var ( - internalErr = errors.New("test internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) -) - // ------------------------ GetExistingResourceNames ------------------------ func TestGetExistingResourceNames(t *testing.T) { etcdBuilder := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testutils.TestNamespace) @@ -58,10 +52,10 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "returns error when client get fails", backupEnabled: true, - getErr: apiInternalErr, + getErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrGetSnapshotLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "GetExistingResourceNames", }, }, @@ -105,10 +99,10 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { }, { name: "returns error when client create fails", - createErr: apiInternalErr, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncSnapshotLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -149,10 +143,10 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { }, { name: "returns error when client delete fails", - deleteAllOfErr: apiInternalErr, + deleteAllOfErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncSnapshotLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "Sync", }, }, @@ -210,10 +204,10 @@ func TestTriggerDelete(t *testing.T) { { name: "should return error when client delete-all fails", backupEnabled: true, - deleteAllErr: apiInternalErr, + deleteAllErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrDeleteSnapshotLease, - Cause: apiInternalErr, + Cause: testutils.TestAPIInternalErr, Operation: "TriggerDelete", }, }, diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index e66cd9ca5..9030743fc 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -112,12 +112,12 @@ func (b *stsBuilder) createStatefulSetObjectMeta() { b.sts.ObjectMeta = metav1.ObjectMeta{ Name: b.etcd.Name, Namespace: b.etcd.Namespace, - Labels: b.getLabels(), + Labels: b.getStatefulSetLabels(), OwnerReferences: []metav1.OwnerReference{b.etcd.GetAsOwnerReference()}, } } -func (b *stsBuilder) getLabels() map[string]string { +func (b *stsBuilder) getStatefulSetLabels() map[string]string { stsLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, druidv1alpha1.LabelAppNameKey: b.etcd.Name, @@ -147,7 +147,7 @@ func (b *stsBuilder) createStatefulSetSpec(ctx resource.OperatorContext) error { ServiceName: b.etcd.GetPeerServiceName(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: utils.MergeMaps[string](b.etcd.Spec.Labels, b.etcd.GetDefaultLabels()), + Labels: utils.MergeMaps[string](b.etcd.Spec.Labels, b.getStatefulSetLabels()), Annotations: b.getPodTemplateAnnotations(ctx), }, Spec: corev1.PodSpec{ @@ -180,9 +180,9 @@ func (b *stsBuilder) getHostAliases() []corev1.HostAlias { } func (b *stsBuilder) getPodTemplateAnnotations(ctx resource.OperatorContext) map[string]string { - if configMapCheckSum, ok := ctx.Data[resource.ConfigMapCheckSumKey]; ok { + if configMapCheckSum, ok := ctx.Data[common.ConfigMapCheckSumKey]; ok { return utils.MergeMaps[string](b.etcd.Spec.Annotations, map[string]string{ - resource.ConfigMapCheckSumKey: configMapCheckSum, + common.ConfigMapCheckSumKey: configMapCheckSum, }) } return b.etcd.Spec.Annotations @@ -215,7 +215,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { return initContainers } initContainers = append(initContainers, corev1.Container{ - Name: "change-permissions", + Name: common.ChangePermissionsInitContainerName, Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, @@ -230,7 +230,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { if b.etcd.IsBackupStoreEnabled() { if b.provider != nil && *b.provider == utils.Local { initContainers = append(initContainers, corev1.Container{ - Name: "change-backup-bucket-permissions", + Name: common.ChangeBackupBucketPermissionsInitContainerName, Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, @@ -283,7 +283,7 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { if b.useEtcdWrapper { return &corev1.VolumeMount{ Name: "host-storage", - MountPath: "/home/nonroot/" + pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, ""), + MountPath: fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, "")), } } else { return &corev1.VolumeMount{ @@ -316,7 +316,7 @@ func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { func (b *stsBuilder) getEtcdContainer() corev1.Container { return corev1.Container{ - Name: "etcd", + Name: common.EtcdContainerName, Image: b.etcdImage, ImagePullPolicy: corev1.PullIfNotPresent, Args: b.getEtcdContainerCommandArgs(), @@ -345,7 +345,7 @@ func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { return corev1.Container{}, err } return corev1.Container{ - Name: "backup-restore", + Name: common.EtcdBackupRestoreContainerName, Image: b.etcdBackupRestoreImage, ImagePullPolicy: corev1.PullIfNotPresent, Args: b.getBackupRestoreContainerCommandArgs(), @@ -746,7 +746,7 @@ func (b *stsBuilder) getBackupVolume(ctx resource.OperatorContext) (*corev1.Volu } store := b.etcd.Spec.Backup.Store switch *b.provider { - case "Local": + case utils.Local: hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, b.client, b.logger, store, b.etcd.GetNamespace()) if err != nil { return nil, fmt.Errorf("error getting host mount path for etcd: %v Err: %w", b.etcd.GetNamespaceName(), err) diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index d1b26344d..4d4aca6b9 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -20,9 +20,9 @@ import ( ) const ( - ErrGetStatefulSet druidv1alpha1.ErrorCode = "ERR_GET_STATEFUL_SET" - ErrDeleteStatefulSet druidv1alpha1.ErrorCode = "ERR_DELETE_STATEFUL_SET" - ErrSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_SYNC_STATEFUL_SET" + ErrGetStatefulSet druidv1alpha1.ErrorCode = "ERR_GET_STATEFULSET" + ErrDeleteStatefulSet druidv1alpha1.ErrorCode = "ERR_DELETE_STATEFULSET" + ErrSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_SYNC_STATEFULSET" ) type _resource struct { @@ -40,12 +40,13 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates } func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { - sts, err := r.getExistingStatefulSet(ctx, etcd) + objectKey := getObjectKey(etcd) + sts, err := r.getExistingStatefulSet(ctx, objectKey) if err != nil { return nil, druiderr.WrapError(err, ErrGetStatefulSet, "GetExistingResourceNames", - fmt.Sprintf("Error getting StatefulSet: %s for etcd: %v", getObjectKey(etcd).Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } if sts == nil { return []string{}, nil @@ -58,18 +59,20 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) existingSTS *appsv1.StatefulSet err error ) - if existingSTS, err = r.getExistingStatefulSet(ctx, etcd); err != nil { + objectKey := getObjectKey(etcd) + if existingSTS, err = r.getExistingStatefulSet(ctx, objectKey); err != nil { return druiderr.WrapError(err, ErrSyncStatefulSet, "Sync", - fmt.Sprintf("Error getting StatefulSet: %s for etcd: %v", getObjectKey(etcd).Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } // There is no StatefulSet present. Create one. if existingSTS == nil { return r.createOrPatch(ctx, etcd) } - + // StatefulSet exists, check if TLS has been enabled for peer communication, if yes then it is currently a multistep + // process to ensure that all members are updated and establish peer TLS communication. if err = r.handlePeerTLSEnabled(ctx, etcd, existingSTS); err != nil { return fmt.Errorf("error handling peer TLS: %w", err) } @@ -80,12 +83,26 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { - return client.IgnoreNotFound(r.client.Delete(ctx, emptyStatefulSet(getObjectKey(etcd)))) + objectKey := getObjectKey(etcd) + ctx.Logger.Info("Triggering delete of StatefulSet", "objectKey", objectKey) + err := r.client.Delete(ctx, emptyStatefulSet(objectKey)) + if err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No StatefulSet found, Deletion is a No-Op", "objectKey", objectKey.Name) + return nil + } + return druiderr.WrapError(err, + ErrDeleteStatefulSet, + "TriggerDelete", + fmt.Sprintf("Failed to delete StatefulSet: %v for etcd %v", objectKey, etcd.GetNamespaceName())) + } + ctx.Logger.Info("deleted", "resource", "statefulset", "objectKey", objectKey) + return nil } -func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { - sts := emptyStatefulSet(getObjectKey(etcd)) - if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { +func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, objectKey client.ObjectKey) (*appsv1.StatefulSet, error) { + sts := emptyStatefulSet(objectKey) + if err := r.client.Get(ctx, objectKey, sts); err != nil { if errors.IsNotFound(err) { return nil, nil } @@ -100,14 +117,20 @@ func (r _resource) createOrPatchWithReplicas(ctx resource.OperatorContext, etcd desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) mutatingFn := func() error { if builder, err := newStsBuilder(r.client, ctx.Logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { - return err + return druiderr.WrapError(err, + ErrSyncStatefulSet, + "Sync", + fmt.Sprintf("Error initializing StatefulSet builder for etcd %v", etcd.GetNamespaceName())) } else { return builder.Build(ctx) } } opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, desiredStatefulSet, mutatingFn) if err != nil { - return err + return druiderr.WrapError(err, + ErrSyncStatefulSet, + "Sync", + fmt.Sprintf("Error creating or patching StatefulSet: %s for etcd: %v", desiredStatefulSet.Name, etcd.GetNamespaceName())) } ctx.Logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) @@ -120,7 +143,7 @@ func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alph } func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - if existingSts.Generation > 1 && hasImmutableFieldChanged(existingSts, etcd) { + if existingSts.Generation > 1 && doesEtcdChangesRequireRecreation(existingSts, etcd) { ctx.Logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") if err := r.TriggerDelete(ctx, etcd); err != nil { @@ -130,10 +153,10 @@ func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etc return nil } -// hasImmutableFieldChanged checks if any immutable fields have changed in the StatefulSet -// specification compared to the Etcd object. It returns true if there are changes in the immutable fields. -// Currently, it checks for changes in the ServiceName and PodManagementPolicy. -func hasImmutableFieldChanged(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { +// doesEtcdChangesRequireRecreation checks if changes have been made to certain fields in the etcd spec +// which will require a recreation of StatefulSet in order to get reflected. Currently, it checks for changes +// in the ServiceName and PodManagementPolicy. +func doesEtcdChangesRequireRecreation(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement } diff --git a/internal/operator/statefulset/statefulset_test.go b/internal/operator/statefulset/statefulset_test.go new file mode 100644 index 000000000..baf1dc780 --- /dev/null +++ b/internal/operator/statefulset/statefulset_test.go @@ -0,0 +1,163 @@ +package statefulset + +import ( + "context" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + druiderr "github.com/gardener/etcd-druid/internal/errors" + "github.com/gardener/etcd-druid/internal/features" + "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/utils" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" + "github.com/google/uuid" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/featuregate" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ------------------------ GetExistingResourceNames ------------------------ +func TestGetExistingResourceNames(t *testing.T) { + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() + testCases := []struct { + name string + stsExists bool + getErr *apierrors.StatusError + expectedStsNames []string + expectedErr *druiderr.DruidError + }{ + { + name: "should return an empty slice if no sts is found", + stsExists: false, + expectedStsNames: []string{}, + }, + { + name: "should return existing sts", + stsExists: true, + expectedStsNames: []string{etcd.Name}, + }, + { + name: "should return err when client get fails", + stsExists: true, + getErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrGetStatefulSet, + Cause: testutils.TestAPIInternalErr, + Operation: "GetExistingResourceNames", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + if tc.stsExists { + fakeClientBuilder.WithObjects(emptyStatefulSet(getObjectKey(etcd))) + } + cl := fakeClientBuilder.Build() + operator := New(cl, nil, nil) + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + actualStsNames, err := operator.GetExistingResourceNames(opCtx, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, err) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(actualStsNames).To(Equal(tc.expectedStsNames)) + } + }) + } +} + +// ----------------------------------- Sync ----------------------------------- +func TestSyncWhenNoSTSExists(t *testing.T) { + testCases := []struct { + name string + replicas int32 + createErr *apierrors.StatusError + expectedErr *druiderr.DruidError + }{ + { + name: "creates a single replica sts for a single node etcd cluster", + replicas: 1, + }, + { + name: "creates multiple replica sts for a multi-node etcd cluster", + replicas: 3, + }, + { + name: "returns error when client create fails", + replicas: 3, + createErr: testutils.TestAPIInternalErr, + expectedErr: &druiderr.DruidError{ + Code: ErrSyncStatefulSet, + Cause: testutils.TestAPIInternalErr, + Operation: "Sync", + }, + }, + } + + g := NewWithT(t) + t.Parallel() + iv := testutils.CreateImageVector(false, false, true, true) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // *************** Build test environment *************** + cl := testutils.NewFakeClientBuilder(). + WithCreateError(tc.createErr). + WithObjects(buildBackupSecret()). + Build() + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(tc.replicas).Build() + etcdImage, etcdBRImage, initContainerImage, err := utils.GetEtcdImages(etcd, iv, true) + g.Expect(err).ToNot(HaveOccurred()) + stsMatcher := testutils.NewStatefulSetMatcher(g, cl, etcd, tc.replicas, true, initContainerImage, etcdImage, etcdBRImage, pointer.String(utils.Local)) + operator := New(cl, iv, map[featuregate.Feature]bool{ + features.UseEtcdWrapper: true, + }) + // *************** Test and assert *************** + opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx.Data[common.ConfigMapCheckSumKey] = testutils.TestConfigMapCheckSum + syncErr := operator.Sync(opCtx, etcd) + latestSTS, getErr := getLatestStatefulSet(cl, etcd) + if tc.expectedErr != nil { + testutils.CheckDruidError(g, tc.expectedErr, syncErr) + g.Expect(apierrors.IsNotFound(getErr)).To(BeTrue()) + } else { + g.Expect(syncErr).ToNot(HaveOccurred()) + g.Expect(getErr).ToNot(HaveOccurred()) + g.Expect(latestSTS).ToNot(BeNil()) + g.Expect(*latestSTS).Should(stsMatcher.MatchStatefulSet()) + } + }) + } +} + +// ----------------------------- TriggerDelete ------------------------------- +// ---------------------------- Helper Functions ----------------------------- + +func getLatestStatefulSet(cl client.Client, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { + sts := &appsv1.StatefulSet{} + err := cl.Get(context.Background(), client.ObjectKeyFromObject(etcd), sts) + return sts, err +} + +func buildBackupSecret() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-backup", + Namespace: testutils.TestNamespace, + }, + Data: map[string][]byte{ + "bucketName": []byte("NDQ5YjEwZj"), + "hostPath": []byte("/var/data/etcd-backup"), + }, + } +} diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index b55b4932b..8a9b29630 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -8,8 +8,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" testutils "github.com/gardener/etcd-druid/test/utils" - "github.com/gardener/gardener/pkg/utils/imagevector" - "k8s.io/utils/pointer" ) // ************************** GetEtcdImages ************************** @@ -35,7 +33,7 @@ func TestGetEtcdImages(t *testing.T) { } func testWithEtcdAndEtcdBRImages(g *WithT, etcd *druidv1alpha1.Etcd) { - iv := createImageVector(true, true, false, false) + iv := testutils.CreateImageVector(true, true, false, false) etcdImg, etcdBRImg, initContainerImg, err := GetEtcdImages(etcd, iv, false) g.Expect(err).To(BeNil()) g.Expect(etcdImg).ToNot(BeEmpty()) @@ -50,7 +48,7 @@ func testWithEtcdAndEtcdBRImages(g *WithT, etcd *druidv1alpha1.Etcd) { func testWithNoImageInSpecAndIVWithEtcdAndBRImages(g *WithT, etcd *druidv1alpha1.Etcd) { etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil - iv := createImageVector(true, true, false, false) + iv := testutils.CreateImageVector(true, true, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) g.Expect(err).To(BeNil()) g.Expect(etcdImage).ToNot(BeEmpty()) @@ -68,7 +66,7 @@ func testWithNoImageInSpecAndIVWithEtcdAndBRImages(g *WithT, etcd *druidv1alpha1 func testSpecWithEtcdBRImageAndIVWithEtcdImage(g *WithT, etcd *druidv1alpha1.Etcd) { etcd.Spec.Etcd.Image = nil - iv := createImageVector(true, false, false, false) + iv := testutils.CreateImageVector(true, false, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) g.Expect(err).To(BeNil()) g.Expect(etcdImage).ToNot(BeEmpty()) @@ -84,7 +82,7 @@ func testSpecWithEtcdBRImageAndIVWithEtcdImage(g *WithT, etcd *druidv1alpha1.Etc func testSpecAndIVWithoutEtcdBRImage(g *WithT, etcd *druidv1alpha1.Etcd) { etcd.Spec.Backup.Image = nil - iv := createImageVector(true, false, false, false) + iv := testutils.CreateImageVector(true, false, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) g.Expect(err).ToNot(BeNil()) g.Expect(etcdImage).To(BeEmpty()) @@ -94,7 +92,7 @@ func testSpecAndIVWithoutEtcdBRImage(g *WithT, etcd *druidv1alpha1.Etcd) { func testWithSpecAndIVNotHavingAnyImages(g *WithT, etcd *druidv1alpha1.Etcd) { etcd.Spec.Backup.Image = nil - iv := createImageVector(false, false, false, false) + iv := testutils.CreateImageVector(false, false, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) g.Expect(err).ToNot(BeNil()) g.Expect(etcdImage).To(BeEmpty()) @@ -105,7 +103,7 @@ func testWithSpecAndIVNotHavingAnyImages(g *WithT, etcd *druidv1alpha1.Etcd) { func testWithNoImagesInSpecAndIVWithAllImagesWithWrapper(g *WithT, etcd *druidv1alpha1.Etcd) { etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil - iv := createImageVector(true, true, true, true) + iv := testutils.CreateImageVector(true, true, true, true) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, true) g.Expect(err).To(BeNil()) g.Expect(etcdImage).ToNot(BeEmpty()) @@ -120,50 +118,3 @@ func testWithNoImagesInSpecAndIVWithAllImagesWithWrapper(g *WithT, etcd *druidv1 g.Expect(err).To(BeNil()) g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) } - -func createImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperImage, withBackupRestoreDistrolessImage bool) imagevector.ImageVector { - var imageSources []*imagevector.ImageSource - const ( - repo = "test-repo" - etcdTag = "etcd-test-tag" - etcdWrapperTag = "etcd-wrapper-test-tag" - backupRestoreTag = "backup-restore-test-tag" - backupRestoreDistrolessTag = "backup-restore-distroless-test-tag" - initContainerTag = "init-container-test-tag" - ) - if withEtcdImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.Etcd, - Repository: repo, - Tag: pointer.String(etcdTag), - }) - } - if withBackupRestoreImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.BackupRestore, - Repository: repo, - Tag: pointer.String(backupRestoreTag), - }) - - } - if withEtcdWrapperImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.EtcdWrapper, - Repository: repo, - Tag: pointer.String(etcdWrapperTag), - }) - } - if withBackupRestoreDistrolessImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.BackupRestoreDistroless, - Repository: repo, - Tag: pointer.String(backupRestoreDistrolessTag), - }) - } - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.Alpine, - Repository: repo, - Tag: pointer.String(initContainerTag), - }) - return imageSources -} diff --git a/test/utils/constants.go b/test/utils/constants.go index db2fbfe1d..7e4ce1dc8 100644 --- a/test/utils/constants.go +++ b/test/utils/constants.go @@ -5,3 +5,24 @@ const ( // TestNamespace is a test namespace to be used in tests. TestNamespace = "test-namespace" ) + +// Image vector constants +const ( + // TestImageRepo is a constant for image repository name + TestImageRepo = "test-repo" + // ETCDImageSourceTag is the ImageSource tag for etcd image. + ETCDImageSourceTag = "etcd-test-tag" + // ETCDWrapperImageTag is the ImageSource tag for etcd-wrapper image. + ETCDWrapperImageTag = "etcd-wrapper-test-tag" + // ETCDBRImageTag is the ImageSource tag for etcd-backup-restore image. + ETCDBRImageTag = "backup-restore-test-tag" + // ETCDBRDistrolessImageTag is the ImageSource tag for etc-backup-restore distroless image. + ETCDBRDistrolessImageTag = "backup-restore-distroless-test-tag" + // InitContainerTag is the ImageSource tag for the init container image. + InitContainerTag = "init-container-test-tag" +) + +const ( + // TestConfigMapCheckSum is a test check-sum for the configmap which is stored as a value against checksum/etcd-configmap annotation put on the etcd sts pods. + TestConfigMapCheckSum = "08ee0a880e10172e337ac57fb20411980a987405a0eaf9a05b8err69ca0f0d69" +) diff --git a/test/utils/errors.go b/test/utils/errors.go index 783d208bc..3a66e7879 100644 --- a/test/utils/errors.go +++ b/test/utils/errors.go @@ -5,6 +5,14 @@ import ( druiderr "github.com/gardener/etcd-druid/internal/errors" . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +var ( + // TestInternalErr is a generic test internal server error used to construct TestAPIInternalErr. + TestInternalErr = errors.New("fake get internal error") + // TestAPIInternalErr is an API internal server error meant to be used to mimic HTTP response code 500 for tests. + TestAPIInternalErr = apierrors.NewInternalError(TestInternalErr) ) // CheckDruidError checks that an actual error is a DruidError and further checks its underline cause, error code and operation. diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 4c63fdc1f..730644496 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -80,7 +80,7 @@ func (eb *EtcdBuilder) WithReplicas(replicas int32) *EtcdBuilder { return eb } -func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { +func (eb *EtcdBuilder) WithClientTLS() *EtcdBuilder { if eb == nil || eb.etcd == nil { return nil } @@ -98,7 +98,15 @@ func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { Name: "client-url-etcd-server-tls", }, } + eb.etcd.Spec.Etcd.ClientUrlTLS = clientTLSConfig + eb.etcd.Spec.Backup.TLS = clientTLSConfig + return eb +} +func (eb *EtcdBuilder) WithPeerTLS() *EtcdBuilder { + if eb == nil || eb.etcd == nil { + return nil + } peerTLSConfig := &druidv1alpha1.TLSConfig{ TLSCASecretRef: druidv1alpha1.SecretReference{ SecretReference: corev1.SecretReference{ @@ -110,11 +118,7 @@ func (eb *EtcdBuilder) WithTLS() *EtcdBuilder { Name: "peer-url-etcd-server-tls", }, } - - eb.etcd.Spec.Etcd.ClientUrlTLS = clientTLSConfig eb.etcd.Spec.Etcd.PeerUrlTLS = peerTLSConfig - eb.etcd.Spec.Backup.TLS = clientTLSConfig - return eb } @@ -343,15 +347,18 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { "role": "test", }, Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: name, + "networking.gardener.cloud/to-dns": "allowed", + "networking.gardener.cloud/to-private-networks": "allowed", + "networking.gardener.cloud/to-public-networks": "allowed", + "networking.gardener.cloud/to-runtime-apiserver": "allowed", + "gardener.cloud/role": "controlplane", }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: name, }, }, Replicas: 1, @@ -367,10 +374,6 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { DefragmentationSchedule: &defragSchedule, EtcdDefragTimeout: &etcdDefragTimeout, Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": ParseQuantity("2500m"), - "memory": ParseQuantity("4Gi"), - }, Requests: corev1.ResourceList{ "cpu": ParseQuantity("500m"), "memory": ParseQuantity("1000Mi"), diff --git a/test/utils/etcdcopybackupstask.go b/test/utils/etcdcopybackupstask.go index e2d5a1067..f2e350c23 100644 --- a/test/utils/etcdcopybackupstask.go +++ b/test/utils/etcdcopybackupstask.go @@ -5,11 +5,11 @@ package utils import ( - "fmt" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,8 +23,8 @@ func CreateEtcdCopyBackupsTask(name, namespace string, provider druidv1alpha1.St waitForFinalSnapshot *druidv1alpha1.WaitForFinalSnapshotSpec ) if withOptionalFields { - maxBackupAge = uint32Ptr(7) - maxBackups = uint32Ptr(42) + maxBackupAge = pointer.Uint32(7) + maxBackups = pointer.Uint32(42) waitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ Enabled: true, Timeout: &metav1.Duration{Duration: 10 * time.Minute}, @@ -64,16 +64,15 @@ func CreateEtcdCopyBackupsTask(name, namespace string, provider druidv1alpha1.St }, } } -func uint32Ptr(v uint32) *uint32 { return &v } // CreateEtcdCopyBackupsJob creates an instance of a Job owned by a EtcdCopyBackupsTask with the given name. func CreateEtcdCopyBackupsJob(taskName, namespace string) *batchv1.Job { + jobName := taskName + "-worker" return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Name: taskName + "-worker", - Namespace: namespace, - Labels: nil, - Annotations: createJobAnnotations(taskName, namespace), + Name: jobName, + Namespace: namespace, + Labels: getLabels(taskName, jobName, false), OwnerReferences: []metav1.OwnerReference{ { APIVersion: druidv1alpha1.GroupVersion.String(), @@ -87,6 +86,9 @@ func CreateEtcdCopyBackupsJob(taskName, namespace string) *batchv1.Job { }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: getLabels(taskName, jobName, true), + }, Spec: corev1.PodSpec{ Volumes: nil, Containers: []corev1.Container{ @@ -104,9 +106,16 @@ func CreateEtcdCopyBackupsJob(taskName, namespace string) *batchv1.Job { } } -func createJobAnnotations(taskName, namespace string) map[string]string { - annotations := make(map[string]string, 2) - annotations[common.GardenerOwnedBy] = fmt.Sprintf("%s/%s", namespace, taskName) - annotations[common.GardenerOwnerType] = "etcdcopybackupstask" - return annotations +func getLabels(taskName, jobName string, includeNetworkPolicyLabels bool) map[string]string { + labels := map[string]string{ + druidv1alpha1.LabelPartOfKey: taskName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelComponentKey: common.EtcdCopyBackupTaskComponentName, + druidv1alpha1.LabelAppNameKey: jobName, + } + if includeNetworkPolicyLabels { + labels[v1beta1constants.LabelNetworkPolicyToDNS] = v1beta1constants.LabelNetworkPolicyAllowed + labels[v1beta1constants.LabelNetworkPolicyToPublicNetworks] = v1beta1constants.LabelNetworkPolicyAllowed + } + return labels } diff --git a/test/utils/imagevector.go b/test/utils/imagevector.go new file mode 100644 index 000000000..e3a1670cf --- /dev/null +++ b/test/utils/imagevector.go @@ -0,0 +1,47 @@ +package utils + +import ( + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + "k8s.io/utils/pointer" +) + +// CreateImageVector creates an image vector initializing it will different image sources. +func CreateImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperImage, withBackupRestoreDistrolessImage bool) imagevector.ImageVector { + var imageSources []*imagevector.ImageSource + if withEtcdImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.Etcd, + Repository: TestImageRepo, + Tag: pointer.String(ETCDImageSourceTag), + }) + } + if withBackupRestoreImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.BackupRestore, + Repository: TestImageRepo, + Tag: pointer.String(ETCDBRImageTag), + }) + + } + if withEtcdWrapperImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.EtcdWrapper, + Repository: TestImageRepo, + Tag: pointer.String(ETCDWrapperImageTag), + }) + } + if withBackupRestoreDistrolessImage { + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.BackupRestoreDistroless, + Repository: TestImageRepo, + Tag: pointer.String(ETCDBRDistrolessImageTag), + }) + } + imageSources = append(imageSources, &imagevector.ImageSource{ + Name: common.Alpine, + Repository: TestImageRepo, + Tag: pointer.String(InitContainerTag), + }) + return imageSources +} diff --git a/test/utils/matcher.go b/test/utils/matcher.go index bee525986..cadacb745 100644 --- a/test/utils/matcher.go +++ b/test/utils/matcher.go @@ -12,19 +12,19 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// mapMatcher matches map[string]string and produces a convenient and easy to consume error message which includes -// the difference between the actual and expected. -type mapMatcher struct { +// mapMatcher matches map[string]T and produces a convenient and easy to consume error message which includes +// the difference between the actual and expected. T is a generic type and accepts any type that is comparable. +type mapMatcher[T comparable] struct { fieldName string - expected map[string]string + expected map[string]T diff []string } -func (m mapMatcher) Match(actual interface{}) (bool, error) { +func (m mapMatcher[T]) Match(actual interface{}) (bool, error) { if actual == nil { return false, nil } - actualMap, okType := actual.(map[string]string) + actualMap, okType := actual.(map[string]T) if !okType { return false, fmt.Errorf("expected a map[string]string. got: %s", format.Object(actual, 1)) } @@ -40,15 +40,15 @@ func (m mapMatcher) Match(actual interface{}) (bool, error) { return len(m.diff) == 0, nil } -func (m mapMatcher) FailureMessage(actual interface{}) string { +func (m mapMatcher[T]) FailureMessage(actual interface{}) string { return m.createMessage(actual, "to be") } -func (m mapMatcher) NegatedFailureMessage(actual interface{}) string { +func (m mapMatcher[T]) NegatedFailureMessage(actual interface{}) string { return m.createMessage(actual, "to not be") } -func (m mapMatcher) createMessage(actual interface{}, message string) string { +func (m mapMatcher[T]) createMessage(actual interface{}, message string) string { msgBuilder := strings.Builder{} msgBuilder.WriteString(format.Message(actual, message, m.expected)) if len(m.diff) > 0 { @@ -60,7 +60,7 @@ func (m mapMatcher) createMessage(actual interface{}, message string) string { // MatchResourceAnnotations returns a custom gomega matcher which matches annotations set on a resource against the expected annotations. func MatchResourceAnnotations(expected map[string]string) gomegatypes.GomegaMatcher { - return &mapMatcher{ + return &mapMatcher[string]{ fieldName: "ObjectMeta.Annotations", expected: expected, } @@ -68,7 +68,7 @@ func MatchResourceAnnotations(expected map[string]string) gomegatypes.GomegaMatc // MatchResourceLabels returns a custom gomega matcher which matches labels set on the resource against the expected labels. func MatchResourceLabels(expected map[string]string) gomegatypes.GomegaMatcher { - return &mapMatcher{ + return &mapMatcher[string]{ fieldName: "ObjectMeta.Labels", expected: expected, } diff --git a/test/utils/stsmatcher.go b/test/utils/stsmatcher.go index d4b585bf7..d44481438 100644 --- a/test/utils/stsmatcher.go +++ b/test/utils/stsmatcher.go @@ -1 +1,540 @@ package utils + +import ( + "context" + "fmt" + "strconv" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + defaultTestContainerResources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse("50m"), + corev1.ResourceMemory: apiresource.MustParse("128Mi"), + }, + } +) + +type StatefulSetMatcher struct { + g *WithT + cl client.Client + replicas int32 + etcd *druidv1alpha1.Etcd + useEtcdWrapper bool + initContainerImage string + etcdImage string + etcdBRImage string + provider *string + clientPort int32 + serverPort int32 + backupPort int32 +} + +func NewStatefulSetMatcher(g *WithT, + cl client.Client, + etcd *druidv1alpha1.Etcd, + replicas int32, + useEtcdWrapper bool, + initContainerImage, etcdImage, etcdBRImage string, + provider *string) StatefulSetMatcher { + return StatefulSetMatcher{ + g: g, + cl: cl, + replicas: replicas, + etcd: etcd, + useEtcdWrapper: useEtcdWrapper, + initContainerImage: initContainerImage, + etcdImage: etcdImage, + etcdBRImage: etcdBRImage, + provider: provider, + clientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, 2379), + serverPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, 2380), + backupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, 8080), + } +} + +func (s StatefulSetMatcher) MatchStatefulSet() gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": s.matchSTSObjectMeta(), + "Spec": s.matchSpec(), + }) +} + +func (s StatefulSetMatcher) matchSTSObjectMeta() gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "Name": Equal(s.etcd.Name), + "Namespace": Equal(s.etcd.Namespace), + "OwnerReferences": MatchEtcdOwnerReference(s.etcd.Name, s.etcd.UID), + "Labels": MatchResourceLabels(getStatefulSetLabels(s.etcd.Name)), + }) +} + +func (s StatefulSetMatcher) matchSpec() gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "Replicas": PointTo(Equal(s.replicas)), + "Selector": MatchSpecLabelSelector(s.etcd.GetDefaultLabels()), + "PodManagementPolicy": Equal(appsv1.ParallelPodManagement), + "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ + "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), + }), + "VolumeClaimTemplates": s.matchVolumeClaimTemplates(), + "ServiceName": Equal(s.etcd.GetPeerServiceName()), + "Template": s.matchPodTemplateSpec(), + }) +} + +func (s StatefulSetMatcher) matchVolumeClaimTemplates() gomegatypes.GomegaMatcher { + defaultStorageCapacity := apiresource.MustParse("16Gi") + return ConsistOf(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name)), + }), + "Spec": MatchFields(IgnoreExtras, Fields{ + "AccessModes": ConsistOf(corev1.ReadWriteOnce), + "Resources": MatchFields(IgnoreExtras, Fields{ + "Requests": MatchKeys(IgnoreExtras, Keys{ + corev1.ResourceStorage: Equal(utils.TypeDeref[apiresource.Quantity](s.etcd.Spec.StorageCapacity, defaultStorageCapacity)), + }), + }), + "StorageClassName": Equal(s.etcd.Spec.StorageClass), + }), + })) +} + +func (s StatefulSetMatcher) matchPodTemplateSpec() gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": s.matchPodObjectMeta(), + "Spec": s.matchPodSpec(), + }) +} + +func (s StatefulSetMatcher) matchPodObjectMeta() gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "Labels": MatchResourceLabels(utils.MergeMaps[string, string](getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), + "Annotations": MatchResourceAnnotations(utils.MergeMaps[string, string](s.etcd.Spec.Annotations, map[string]string{ + "checksum/etcd-configmap": TestConfigMapCheckSum, + })), + }) +} + +func (s StatefulSetMatcher) matchPodSpec() gomegatypes.GomegaMatcher { + // NOTE: currently this matcher does not check affinity and TSC since these are seldom used. If these are used in future then this matcher should be enhanced. + return MatchFields(IgnoreExtras, Fields{ + "HostAliases": s.matchPodHostAliases(), + "ServiceAccountName": Equal(s.etcd.GetServiceAccountName()), + "ShareProcessNamespace": PointTo(Equal(true)), + "InitContainers": s.matchPodInitContainers(), + "Containers": s.matchContainers(), + "SecurityContext": s.matchEtcdPodSecurityContext(), + "Volumes": s.matchPodVolumes(), + "PriorityClassName": Equal(utils.TypeDeref[string](s.etcd.Spec.PriorityClassName, "")), + }) +} + +func (s StatefulSetMatcher) matchPodHostAliases() gomegatypes.GomegaMatcher { + return ConsistOf(MatchFields(IgnoreExtras, Fields{ + "IP": Equal("127.0.0.1"), + "Hostnames": ConsistOf(fmt.Sprintf("%s-local", s.etcd.Name)), + })) +} + +func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { + if !s.useEtcdWrapper { + return BeEmpty() + } + initContainerMatcherElements := make(map[string]gomegatypes.GomegaMatcher) + volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) + changePermissionsInitContainerMatcher := MatchFields(IgnoreExtras, Fields{ + "Name": Equal("change-permissions"), + "Image": Equal(s.initContainerImage), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + "Command": HaveExactElements("sh", "-c", "--"), + "Args": HaveExactElements("chown -R 65532:65532 /var/etcd/data"), + "VolumeMounts": ConsistOf(MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumeClaimTemplateName), + "MountPath": Equal("/var/etcd/data"), + })), + "SecurityContext": matchPodInitContainerSecurityContext(), + }) + initContainerMatcherElements["change-permissions"] = changePermissionsInitContainerMatcher + if s.etcd.IsBackupStoreEnabled() && s.provider != nil && *s.provider == utils.Local { + changeBackupBucketPermissionsMatcher := MatchFields(IgnoreExtras, Fields{ + "Name": Equal("change-backup-bucket-permissions"), + "Image": Equal(s.initContainerImage), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + "Command": HaveExactElements("sh", "-c", "--"), + "Args": HaveExactElements(fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *s.etcd.Spec.Backup.Store.Container)), + "VolumeMounts": s.matchBackupRestoreContainerVolMounts(), + "SecurityContext": matchPodInitContainerSecurityContext(), + }) + initContainerMatcherElements["change-backup-bucket-permissions"] = changeBackupBucketPermissionsMatcher + } + return MatchAllElements(containerIdentifier, initContainerMatcherElements) +} + +func (s StatefulSetMatcher) matchContainers() gomegatypes.GomegaMatcher { + return MatchAllElements(containerIdentifier, Elements{ + common.EtcdContainerName: s.matchEtcdContainer(), + common.EtcdBackupRestoreContainerName: s.matchBackupRestoreContainer(), + }) +} + +func (s StatefulSetMatcher) matchEtcdContainer() gomegatypes.GomegaMatcher { + etcdContainerResources := utils.TypeDeref[corev1.ResourceRequirements](s.etcd.Spec.Etcd.Resources, defaultTestContainerResources) + return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Name": Equal("etcd"), + "Image": Equal(s.etcdImage), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + "Args": s.matchEtcdContainerCmdArgs(), + "ReadinessProbe": PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "ProbeHandler": s.matchEtcdContainerReadinessHandler(), + "InitialDelaySeconds": Equal(int32(15)), + "PeriodSeconds": Equal(int32(5)), + "FailureThreshold": Equal(int32(5)), + })), + "Ports": ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("server"), + "Protocol": Equal(corev1.ProtocolTCP), + "ContainerPort": Equal(s.serverPort), + }), + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client"), + "Protocol": Equal(corev1.ProtocolTCP), + "ContainerPort": Equal(s.clientPort), + }), + ), + "Resources": Equal(etcdContainerResources), + "Env": s.matchEtcdContainerEnvVars(), + "VolumeMounts": s.matchEtcdContainerVolMounts(), + }) +} + +func (s StatefulSetMatcher) matchEtcdContainerVolMounts() gomegatypes.GomegaMatcher { + volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) + volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) + secretVolMountMatchers := s.getSecretVolMountsMatchers() + if secretVolMountMatchers != nil && len(secretVolMountMatchers) > 0 { + volMountMatchers = append(volMountMatchers, secretVolMountMatchers...) + } + return ConsistOf(volMountMatchers) +} + +func (s StatefulSetMatcher) matchBackupRestoreContainer() gomegatypes.GomegaMatcher { + containerResources := TypeDeref(s.etcd.Spec.Backup.Resources, defaultTestContainerResources) + return MatchFields(IgnoreExtras, Fields{ + "Name": Equal("backup-restore"), + "Image": Equal(s.etcdBRImage), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + // This is quite painful and therefore skipped for now. Independent unit test for command line args should be written instead. + //"Args": Equal(s.matchBackupRestoreContainerCmdArgs()), + "Ports": ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("server"), + "Protocol": Equal(corev1.ProtocolTCP), + "ContainerPort": Equal(s.backupPort), + }), + ), + "Resources": Equal(containerResources), + "VolumeMounts": s.matchBackupRestoreContainerVolMounts(), + "SecurityContext": PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Capabilities": PointTo(MatchFields(IgnoreExtras, Fields{ + "Add": ConsistOf(corev1.Capability("SYS_PTRACE")), + })), + })), + }) +} + +func (s StatefulSetMatcher) matchEtcdContainerReadinessHandler() gomegatypes.GomegaMatcher { + if s.etcd.Spec.Replicas > 1 && !s.useEtcdWrapper { + return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Exec": PointTo(MatchFields(IgnoreExtras, Fields{ + "Command": s.matchEtcdContainerReadinessProbeCmd(), + })), + }) + } + scheme := utils.IfConditionOr[corev1.URIScheme](s.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) + path := utils.IfConditionOr[string](s.etcd.Spec.Replicas > 1, "/readyz", "/healthz") + port := utils.IfConditionOr[int32](s.etcd.Spec.Replicas > 1, 9095, 8080) + return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "HTTPGet": PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Path": Equal(path), + "Port": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Type": Equal(intstr.Int), + "IntVal": Equal(port), + }), + "Scheme": Equal(scheme), + })), + }) +} + +func (s StatefulSetMatcher) matchEtcdContainerReadinessProbeCmd() gomegatypes.GomegaMatcher { + if s.etcd.Spec.Etcd.ClientUrlTLS != nil { + dataKey := utils.TypeDeref(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") + return HaveExactElements( + "/bin/sh", + "-ec", + fmt.Sprintf("ETCDCTL_API=3 etcdctl --cacert=/var/etcd/ssl/client/ca/%s --cert=/var/etcd/ssl/client/client/tls.crt --key=/var/etcd/ssl/client/client/tls.key --endpoints=https://%s-local:%d get foo --consistency=l", dataKey, s.etcd.Name, s.clientPort), + ) + } else { + return HaveExactElements( + "/bin/sh", + "-ec", + fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=http://%s-local:%d get foo --consistency=l", s.etcd.Name, s.clientPort), + ) + } +} + +func (s StatefulSetMatcher) matchEtcdContainerCmdArgs() gomegatypes.GomegaMatcher { + if !s.useEtcdWrapper { + return BeEmpty() + } + cmdArgs := make([]string, 0, 8) + cmdArgs = append(cmdArgs, "start-etcd") + cmdArgs = append(cmdArgs, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", s.etcd.Name)) + cmdArgs = append(cmdArgs, fmt.Sprintf("--etcd-server-name=%s-local", s.etcd.Name)) + if s.etcd.Spec.Etcd.ClientUrlTLS == nil { + cmdArgs = append(cmdArgs, "--backup-restore-tls-enabled=false") + } else { + dataKey := utils.TypeDeref(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") + cmdArgs = append(cmdArgs, "--backup-restore-tls-enabled=true") + cmdArgs = append(cmdArgs, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") + cmdArgs = append(cmdArgs, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") + cmdArgs = append(cmdArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) + } + return HaveExactElements(cmdArgs) +} + +func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { + volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) + return matchVolMount(volumeClaimTemplateName, "/var/etcd/data") +} + +func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.GomegaMatcher { + volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 8) + volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) + volMountMatchers = append(volMountMatchers, matchVolMount("etcd-config-file", "/var/etcd/config/")) + volMountMatchers = append(volMountMatchers, s.getSecretVolMountsMatchers()...) + if s.etcd.IsBackupStoreEnabled() { + etcdBackupVolMountMatcher := s.getEtcdBackupVolumeMountMatcher() + if etcdBackupVolMountMatcher != nil { + volMountMatchers = append(volMountMatchers, etcdBackupVolMountMatcher) + } + } + return ConsistOf(volMountMatchers) +} + +func (s StatefulSetMatcher) getSecretVolMountsMatchers() []gomegatypes.GomegaMatcher { + secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) + if s.etcd.Spec.Etcd.ClientUrlTLS != nil { + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("client-url-ca-etcd", "/var/etcd/ssl/client/ca")) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("client-url-etcd-server-tls", "/var/etcd/ssl/client/server")) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("client-url-etcd-client-tls", "/var/etcd/ssl/client/client")) + } + if s.etcd.Spec.Etcd.PeerUrlTLS != nil { + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("peer-url-ca-etcd", "/var/etcd/ssl/peer/ca")) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("", "/var/etcd/ssl/peer/server")) + } + return secretVolMountMatchers +} + +func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.GomegaMatcher { + switch *s.provider { + case utils.Local: + if s.etcd.Spec.Backup.Store.Container != nil { + if s.useEtcdWrapper { + return matchVolMount("host-storage", "/home/nonroot/"+pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) + } else { + return matchVolMount("host-storage", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) + } + } + case utils.GCS: + return matchVolMount("etcd-backup", "/var/.gcp/") + case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + return matchVolMount("etcd-backup", "/var/etcd-backup/") + } + return nil +} + +func (s StatefulSetMatcher) matchEtcdContainerEnvVars() gomegatypes.GomegaMatcher { + if s.useEtcdWrapper { + return BeEmpty() + } + scheme := utils.IfConditionOr[string](s.etcd.Spec.Backup.TLS != nil, "https", "http") + return ConsistOf( + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ENABLE_TLS"), + "Value": Equal(strconv.FormatBool(s.etcd.Spec.Backup.TLS != nil)), + }), + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("BACKUP_ENDPOINT"), + "Value": Equal(fmt.Sprintf("%s://%s-local:%d", scheme, s.etcd.Name, s.backupPort)), + }), + ) +} + +func (s StatefulSetMatcher) matchEtcdPodSecurityContext() gomegatypes.GomegaMatcher { + if !s.useEtcdWrapper { + Equal(nil) + } + return PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "RunAsGroup": PointTo(Equal(int64(65532))), + "RunAsNonRoot": PointTo(Equal(true)), + "RunAsUser": PointTo(Equal(int64(65532))), + })) +} + +func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { + volMatchers := make([]gomegatypes.GomegaMatcher, 0, 7) + etcdConfigFileVolMountMatcher := MatchFields(IgnoreExtras, Fields{ + "Name": Equal("etcd-config-file"), + "VolumeSource": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ + "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(s.etcd.GetConfigMapName()), + }), + "Items": HaveExactElements(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Key": Equal("etcd.conf.yaml"), + "Path": Equal("etcd.conf.yaml"), + })), + "DefaultMode": PointTo(Equal(int32(0644))), + })), + }), + }) + volMatchers = append(volMatchers, etcdConfigFileVolMountMatcher) + secretVolMatchers := s.getPodSecurityVolumeMatchers() + if secretVolMatchers != nil && len(secretVolMatchers) > 0 { + volMatchers = append(volMatchers, secretVolMatchers...) + } + if s.etcd.IsBackupStoreEnabled() { + backupVolMatcher := s.getBackupVolumeMatcher() + if backupVolMatcher != nil { + volMatchers = append(volMatchers, backupVolMatcher) + } + } + return ConsistOf(volMatchers) +} + +func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaMatcher { + volMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) + if s.etcd.Spec.Etcd.ClientUrlTLS != nil { + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-ca-etcd"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), + })), + }), + })) + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-server-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), + })), + }), + })) + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-client-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), + })), + }), + })) + } + if s.etcd.Spec.Etcd.PeerUrlTLS != nil { + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-ca-etcd"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), + })), + }), + })) + + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-etcd-server-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), + })), + }), + })) + } + return volMatchers +} + +func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { + if s.provider == nil { + return nil + } + switch *s.provider { + case utils.Local: + hostPath, err := utils.GetHostMountPathFromSecretRef(context.Background(), s.cl, logr.Discard(), s.etcd.Spec.Backup.Store, s.etcd.Namespace) + s.g.Expect(err).ToNot(HaveOccurred()) + return MatchFields(IgnoreExtras, Fields{ + "Name": Equal("host-storage"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "HostPath": PointTo(MatchFields(IgnoreExtras, Fields{ + "Path": Equal(fmt.Sprintf("%s/%s", hostPath, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))), + "Type": PointTo(Equal(corev1.HostPathDirectory)), + })), + }), + }) + + case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + s.g.Expect(s.etcd.Spec.Backup.Store.SecretRef).ToNot(BeNil()) + return MatchFields(IgnoreExtras, Fields{ + "Name": Equal("etcd-backup"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Backup.Store.SecretRef.Name), + })), + }), + }) + } + return nil +} + +func matchVolMount(name, mountPath string) gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "Name": Equal(name), + "MountPath": Equal(mountPath), + }) +} + +func matchPodInitContainerSecurityContext() gomegatypes.GomegaMatcher { + return PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ + "RunAsGroup": PointTo(Equal(int64(0))), + "RunAsNonRoot": PointTo(Equal(false)), + "RunAsUser": PointTo(Equal(int64(0))), + })) +} + +func getStatefulSetLabels(etcdName string) map[string]string { + return map[string]string{ + druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelAppNameKey: etcdName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: etcdName, + } +} + +func containerIdentifier(element interface{}) string { + return (element.(corev1.Container)).Name +} From 428ac72677bb01f1cdf20a1c3c7fa9733edf20ac Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 29 Jan 2024 15:24:34 +0530 Subject: [PATCH 083/235] fixed make check errors --- pkg/utils/image_test.go | 12 ++++++------ pkg/utils/statefulset_test.go | 17 ++++++++--------- .../controllers/compaction/reconciler_test.go | 11 +++++------ .../controllers/custodian/reconciler_test.go | 7 +++---- .../controllers/etcd/reconciler_test.go | 15 +++++++-------- .../controllers/secret/reconciler_test.go | 9 ++++----- test/utils/matcher.go | 1 + test/utils/stsmatcher.go | 3 +++ 8 files changed, 37 insertions(+), 38 deletions(-) diff --git a/pkg/utils/image_test.go b/pkg/utils/image_test.go index 9c0109d3b..e0f61a336 100644 --- a/pkg/utils/image_test.go +++ b/pkg/utils/image_test.go @@ -7,7 +7,7 @@ package utils import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" - testsample "github.com/gardener/etcd-druid/test/sample" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/ginkgo/v2" @@ -28,7 +28,7 @@ var _ = Describe("Image retrieval tests", func() { ) It("etcd spec defines etcd and backup-restore images", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() imageVector = createImageVector(true, true, false, false) etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) Expect(err).To(BeNil()) @@ -42,7 +42,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec has no image defined and image vector has both images set", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil @@ -63,7 +63,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil imageVector = createImageVector(true, false, false, false) @@ -81,7 +81,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("both spec and image vector do not have backup-restore image", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Backup.Image = nil imageVector = createImageVector(true, false, false, false) @@ -93,7 +93,7 @@ var _ = Describe("Image retrieval tests", func() { }) It("etcd spec has no images defined, image vector has all images, and UseEtcdWrapper feature gate is turned on", func() { - etcd = testsample.EtcdBuilderWithDefaults(etcdName, namespace).Build() + etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() Expect(err).To(BeNil()) etcd.Spec.Etcd.Image = nil etcd.Spec.Backup.Image = nil diff --git a/pkg/utils/statefulset_test.go b/pkg/utils/statefulset_test.go index a9c69f3c0..a7df321e5 100644 --- a/pkg/utils/statefulset_test.go +++ b/pkg/utils/statefulset_test.go @@ -9,8 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/client/kubernetes" - testsample "github.com/gardener/etcd-druid/test/sample" - "github.com/gardener/etcd-druid/test/utils" + testutils "github.com/gardener/etcd-druid/test/utils" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -32,7 +31,7 @@ var _ = Describe("tests for statefulset utility functions", func() { stsNamespace = "test-ns" ) It("statefulset has less number of ready replicas as compared to configured etcd replicas", func() { - sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 2) + sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 2) sts.Generation = 1 sts.Status.ObservedGeneration = 1 sts.Status.Replicas = 2 @@ -43,7 +42,7 @@ var _ = Describe("tests for statefulset utility functions", func() { }) It("statefulset has equal number of replicas as defined in etcd but observed generation is outdated", func() { - sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) + sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) sts.Generation = 2 sts.Status.ObservedGeneration = 1 sts.Status.Replicas = 3 @@ -54,8 +53,8 @@ var _ = Describe("tests for statefulset utility functions", func() { }) It("statefulset has equal number of replicas as defined in etcd and observed generation = generation", func() { - sts := utils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) - utils.SetStatefulSetReady(sts) + sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) + testutils.SetStatefulSetReady(sts) ready, reasonMsg := IsStatefulSetReady(3, sts) Expect(ready).To(BeTrue()) Expect(len(reasonMsg)).To(BeZero()) @@ -75,7 +74,7 @@ var _ = Describe("tests for statefulset utility functions", func() { BeforeEach(func() { ctx = context.TODO() stsListToCleanup = &appsv1.StatefulSetList{} - etcd = testsample.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() + etcd = testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() }) AfterEach(func() { @@ -91,7 +90,7 @@ var _ = Describe("tests for statefulset utility functions", func() { }) It("statefulset is present but it is not owned by etcd", func() { - sts := utils.CreateStatefulSet(etcd.Name, etcd.Namespace, uuid.NewUUID(), 3) + sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, uuid.NewUUID(), 3) Expect(fakeClient.Create(ctx, sts)).To(Succeed()) stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) @@ -100,7 +99,7 @@ var _ = Describe("tests for statefulset utility functions", func() { }) It("found statefulset owned by etcd", func() { - sts := utils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, 3) + sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, 3) Expect(fakeClient.Create(ctx, sts)).To(Succeed()) stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index 79453a2fa..ad7ed76e0 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -12,7 +12,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" "github.com/gardener/etcd-druid/pkg/utils" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -48,7 +47,7 @@ var _ = Describe("Compaction Controller", func() { j *batchv1.Job ) - instance = testsample.EtcdBuilderWithDefaults(name, namespace).WithTLS().WithStorageProvider(provider).Build() + instance = testutils.EtcdBuilderWithDefaults(name, namespace).WithPeerTLS().WithClientTLS().WithStorageProvider(provider).Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -103,7 +102,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testsample.EtcdBuilderWithDefaults("foo77", namespace).WithProviderLocal().Build() + instance = testutils.EtcdBuilderWithDefaults("foo77", namespace).WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -155,7 +154,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testsample.EtcdBuilderWithDefaults("foo78", "default").WithProviderLocal().Build() + instance = testutils.EtcdBuilderWithDefaults("foo78", "default").WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -199,7 +198,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testsample.EtcdBuilderWithDefaults("foo79", "default").WithProviderLocal().Build() + instance = testutils.EtcdBuilderWithDefaults("foo79", "default").WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running @@ -240,7 +239,7 @@ var _ = Describe("Compaction Controller", func() { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() - instance = testsample.EtcdBuilderWithDefaults("foo80", "default").WithProviderLocal().Build() + instance = testutils.EtcdBuilderWithDefaults("foo80", "default").WithProviderLocal().Build() createEtcdAndWait(k8sClient, instance) // manually create full and delta snapshot leases since etcd controller is not running diff --git a/test/integration/controllers/custodian/reconciler_test.go b/test/integration/controllers/custodian/reconciler_test.go index c4fcb0007..fb1f4ab1a 100644 --- a/test/integration/controllers/custodian/reconciler_test.go +++ b/test/integration/controllers/custodian/reconciler_test.go @@ -11,7 +11,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" componentpdb "github.com/gardener/etcd-druid/pkg/component/etcd/poddisruptionbudget" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -39,7 +38,7 @@ var _ = Describe("Custodian Controller", func() { ) BeforeEach(func() { - instance = testsample.EtcdBuilderWithDefaults(name, namespace).Build() + instance = testutils.EtcdBuilderWithDefaults(name, namespace).Build() Expect(k8sClient.Create(ctx, instance)).To(Succeed()) // wait for Etcd creation to succeed @@ -165,7 +164,7 @@ var _ = Describe("Custodian Controller", func() { Describe("PodDisruptionBudget", func() { Context("minAvailable of PodDisruptionBudget", func() { When("having a single node cluster", func() { - etcd := testsample.EtcdBuilderWithDefaults("test", "default").WithReadyStatus().Build() + etcd := testutils.EtcdBuilderWithDefaults("test", "default").WithReadyStatus().Build() Expect(len(etcd.Status.Members)).To(BeEquivalentTo(1)) @@ -178,7 +177,7 @@ var _ = Describe("Custodian Controller", func() { }) When("having a multi node cluster", func() { - etcd := testsample.EtcdBuilderWithDefaults("test", "default").WithReplicas(3).WithReadyStatus().Build() + etcd := testutils.EtcdBuilderWithDefaults("test", "default").WithReplicas(3).WithReadyStatus().Build() Expect(len(etcd.Status.Members)).To(BeEquivalentTo(3)) diff --git a/test/integration/controllers/etcd/reconciler_test.go b/test/integration/controllers/etcd/reconciler_test.go index fbabccae2..123110fbc 100644 --- a/test/integration/controllers/etcd/reconciler_test.go +++ b/test/integration/controllers/etcd/reconciler_test.go @@ -16,7 +16,6 @@ import ( "github.com/gardener/etcd-druid/pkg/common" "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/etcd-druid/test/integration/controllers/assets" - testsample "github.com/gardener/etcd-druid/test/sample" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" gardenerUtils "github.com/gardener/gardener/pkg/utils" @@ -80,7 +79,7 @@ var _ = Describe("Etcd Controller", func() { ) BeforeEach(func() { - instance = testsample.EtcdBuilderWithDefaults("foo1", namespace).Build() + instance = testutils.EtcdBuilderWithDefaults("foo1", namespace).Build() storeSecret := instance.Spec.Backup.Store.SecretRef.Name errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) @@ -197,15 +196,15 @@ var _ = Describe("Etcd Controller", func() { rb *rbac.RoleBinding ctx = context.TODO() instance *druidv1alpha1.Etcd - instanceBuilder *testsample.EtcdBuilder + instanceBuilder *testutils.EtcdBuilder ) if etcdWithDefaults { - instanceBuilder = testsample.EtcdBuilderWithDefaults(etcdName, namespace) + instanceBuilder = testutils.EtcdBuilderWithDefaults(etcdName, namespace) } else { - instanceBuilder = testsample.EtcdBuilderWithoutDefaults(etcdName, namespace) + instanceBuilder = testutils.EtcdBuilderWithoutDefaults(etcdName, namespace) } if withTLS { - instanceBuilder.WithTLS() + instanceBuilder.WithPeerTLS().WithClientTLS() } instance = instanceBuilder.WithStorageProvider(provider).Build() @@ -279,7 +278,7 @@ var _ = Describe("Multinode ETCD", func() { ) BeforeEach(func() { - instance = testsample.EtcdBuilderWithDefaults("foo82", namespace).Build() + instance = testutils.EtcdBuilderWithDefaults("foo82", namespace).Build() storeSecret := instance.Spec.Backup.Store.SecretRef.Name errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) Expect(len(errors)).Should(BeZero()) @@ -380,7 +379,7 @@ var _ = Describe("Multinode ETCD", func() { ctx = context.TODO() instance *druidv1alpha1.Etcd ) - instance = testsample.EtcdBuilderWithDefaults(etcdName, namespace).WithReplicas(int32(replicas)).Build() + instance = testutils.EtcdBuilderWithDefaults(etcdName, namespace).WithReplicas(int32(replicas)).Build() if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { storeSecret := instance.Spec.Backup.Store.SecretRef.Name diff --git a/test/integration/controllers/secret/reconciler_test.go b/test/integration/controllers/secret/reconciler_test.go index a3ac80a10..f320a63d8 100644 --- a/test/integration/controllers/secret/reconciler_test.go +++ b/test/integration/controllers/secret/reconciler_test.go @@ -10,8 +10,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" - testsample "github.com/gardener/etcd-druid/test/sample" - "github.com/gardener/etcd-druid/test/utils" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/test/matchers" . "github.com/onsi/ginkgo/v2" @@ -33,7 +32,7 @@ var _ = Describe("Secret Controller", func() { ) BeforeEach(func() { - etcd = testsample.EtcdBuilderWithDefaults("etcd", namespace).WithTLS().Build() + etcd = testutils.EtcdBuilderWithDefaults("etcd", namespace).WithClientTLS().WithPeerTLS().Build() }) It("should reconcile the finalizers for the referenced secrets", func() { @@ -54,7 +53,7 @@ var _ = Describe("Secret Controller", func() { "peer-url-etcd-server-tls", "etcd-backup", } - errs := utils.CreateSecrets(ctx, k8sClient, namespace, secretNames...) + errs := testutils.CreateSecrets(ctx, k8sClient, namespace, secretNames...) Expect(errs).To(BeEmpty()) Expect(k8sClient.Create(ctx, etcd)).To(Succeed()) @@ -73,7 +72,7 @@ var _ = Describe("Secret Controller", func() { "peer-url-etcd-server-tls2", "etcd-backup2", } - errs = utils.CreateSecrets(ctx, k8sClient, namespace, newSecretNames...) + errs = testutils.CreateSecrets(ctx, k8sClient, namespace, newSecretNames...) Expect(errs).To(BeEmpty()) patch := client.MergeFrom(etcd.DeepCopy()) diff --git a/test/utils/matcher.go b/test/utils/matcher.go index cadacb745..abffd0b48 100644 --- a/test/utils/matcher.go +++ b/test/utils/matcher.go @@ -74,6 +74,7 @@ func MatchResourceLabels(expected map[string]string) gomegatypes.GomegaMatcher { } } +// MatchSpecLabelSelector returns a custom gomega matcher which matches label selector on a resource against the expected labels. func MatchSpecLabelSelector(expected map[string]string) gomegatypes.GomegaMatcher { return PointTo(MatchFields(IgnoreExtras, Fields{ "MatchLabels": MatchResourceLabels(expected), diff --git a/test/utils/stsmatcher.go b/test/utils/stsmatcher.go index d44481438..1f34e736e 100644 --- a/test/utils/stsmatcher.go +++ b/test/utils/stsmatcher.go @@ -29,6 +29,7 @@ var ( } ) +// StatefulSetMatcher is the type used for matching StatefulSets. It holds relevant information required for matching. type StatefulSetMatcher struct { g *WithT cl client.Client @@ -44,6 +45,7 @@ type StatefulSetMatcher struct { backupPort int32 } +// NewStatefulSetMatcher constructs a new instance of StatefulSetMatcher. func NewStatefulSetMatcher(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd, @@ -67,6 +69,7 @@ func NewStatefulSetMatcher(g *WithT, } } +// MatchStatefulSet returns a custom gomega matcher that will match both the ObjectMeta and Spec of a StatefulSet. func (s StatefulSetMatcher) MatchStatefulSet() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "ObjectMeta": s.matchSTSObjectMeta(), From 4075ceddcdd2757244e9d09c5e6dac4520d0b348 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 6 Feb 2024 12:39:25 +0530 Subject: [PATCH 084/235] added 2 revised IT tests, misc refactorings all over the place --- internal/controller/etcd/reconcile_delete.go | 18 +- internal/controller/etcd/reconcile_spec.go | 18 +- internal/controller/etcd/reconcile_status.go | 73 +++++ internal/controller/etcd/reconciler.go | 85 ++---- internal/controller/utils/etcdstatus.go | 16 +- internal/errors/errors.go | 3 +- internal/health/condition/builder.go | 2 +- .../health/condition/check_backup_ready.go | 4 +- .../condition/check_data_volumes_ready.go | 2 +- .../operator/clientservice/clientservice.go | 18 +- .../clientservice/clientservice_test.go | 10 +- .../operator/{resource => component}/types.go | 6 +- internal/operator/configmap/configmap.go | 18 +- internal/operator/configmap/configmap_test.go | 92 ++++++- internal/operator/memberlease/memberlease.go | 20 +- .../operator/memberlease/memberlease_test.go | 8 +- internal/operator/peerservice/peerservice.go | 18 +- .../operator/peerservice/peerservice_test.go | 10 +- .../poddisruptionbudget.go | 18 +- .../poddisruptionbudget_test.go | 10 +- internal/operator/registry.go | 86 ++---- internal/operator/role/role.go | 21 +- internal/operator/role/role_test.go | 8 +- internal/operator/rolebinding/rolebinding.go | 23 +- .../operator/rolebinding/rolebinding_test.go | 8 +- .../operator/serviceaccount/serviceaccount.go | 18 +- .../serviceaccount/serviceaccount_test.go | 8 +- .../operator/snapshotlease/snapshotlease.go | 33 +-- .../snapshotlease/snapshotlease_test.go | 10 +- internal/operator/statefulset/builder.go | 12 +- internal/operator/statefulset/statefulset.go | 89 +++--- .../operator/statefulset/statefulset_test.go | 6 +- internal/utils/concurrent.go | 8 +- internal/utils/statefulset.go | 5 +- .../controllers/etcd/reconciler_test.go | 16 +- test/it/controller/assets/assets.go | 52 ++++ test/it/controller/etcd/assertions.go | 257 ++++++++++++++++++ test/it/controller/etcd/reconciler_test.go | 181 ++++++++++++ test/it/setup/setup.go | 127 +++++++++ test/utils/constants.go | 4 +- test/utils/etcd.go | 7 + test/utils/secret.go | 9 +- 42 files changed, 1071 insertions(+), 366 deletions(-) create mode 100644 internal/controller/etcd/reconcile_status.go rename internal/operator/{resource => component}/types.go (91%) create mode 100644 test/it/controller/assets/assets.go create mode 100644 test/it/controller/etcd/assertions.go create mode 100644 test/it/controller/etcd/reconciler_test.go create mode 100644 test/it/setup/setup.go diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 5e6eae694..5cda2510f 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -7,15 +7,15 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" ) -// triggerDeletionFlow is the entry point for the deletion flow triggered for an etcd resource which has a DeletionTimeStamp set on it. -func (r *Reconciler) triggerDeletionFlow(ctx resource.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { +// triggerDeletionFlow is the entry point for the deletion flow triggered for an etcd component which has a DeletionTimeStamp set on it. +func (r *Reconciler) triggerDeletionFlow(ctx component.OperatorContext, logger logr.Logger, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { deleteStepFns := []reconcileFn{ r.recordDeletionStartOperation, r.deleteEtcdResources, @@ -30,7 +30,7 @@ func (r *Reconciler) triggerDeletionFlow(ctx resource.OperatorContext, logger lo return ctrlutils.DoNotRequeue() } -func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -41,7 +41,7 @@ func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, etcdObjKe operator := operator deleteTasks = append(deleteTasks, utils.OperatorTask{ Name: fmt.Sprintf("triggerDeletionFlow-%s-operator", kind), - Fn: func(ctx resource.OperatorContext) error { + Fn: func(ctx component.OperatorContext) error { return operator.TriggerDelete(ctx, etcd) }, }) @@ -53,7 +53,7 @@ func (r *Reconciler) deleteEtcdResources(ctx resource.OperatorContext, etcdObjKe return ctrlutils.ContinueReconcile() } -func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -75,7 +75,7 @@ func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx resource.OperatorContext, return ctrlutils.ContinueReconcile() } -func (r *Reconciler) removeFinalizer(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) removeFinalizer(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -87,7 +87,7 @@ func (r *Reconciler) removeFinalizer(ctx resource.OperatorContext, etcdObjKey cl return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordDeletionStartOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordDeletionStartOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -99,7 +99,7 @@ func (r *Reconciler) recordDeletionStartOperation(ctx resource.OperatorContext, return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordIncompleteDeletionOperation(ctx resource.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordIncompleteDeletionOperation(ctx component.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index c852822c4..0cefee2a0 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -19,7 +19,7 @@ import ( "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/operator" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func (r *Reconciler) triggerReconcileSpecFlow(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { reconcileStepFns := []reconcileFn{ r.recordReconcileStartOperation, r.syncEtcdResources, @@ -45,7 +45,7 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx resource.OperatorContext, etcd return ctrlutils.ContinueReconcile() } -func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -60,7 +60,7 @@ func (r *Reconciler) removeOperationAnnotation(ctx resource.OperatorContext, etc } return ctrlutils.ContinueReconcile() } -func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -75,7 +75,7 @@ func (r *Reconciler) syncEtcdResources(ctx resource.OperatorContext, etcdObjKey return ctrlutils.ContinueReconcile() } -func (r *Reconciler) updateObservedGeneration(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) updateObservedGeneration(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -89,7 +89,7 @@ func (r *Reconciler) updateObservedGeneration(ctx resource.OperatorContext, etcd return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordReconcileStartOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile start operation") return ctrlutils.ReconcileWithError(err) @@ -97,7 +97,7 @@ func (r *Reconciler) recordReconcileStartOperation(ctx resource.OperatorContext, return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordReconcileSuccessOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { if err := r.lastOpErrRecorder.RecordSuccess(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil { ctx.Logger.Error(err, "failed to record etcd reconcile success operation") return ctrlutils.ReconcileWithError(err) @@ -105,9 +105,9 @@ func (r *Reconciler) recordReconcileSuccessOperation(ctx resource.OperatorContex return ctrlutils.ContinueReconcile() } -func (r *Reconciler) recordIncompleteReconcileOperation(ctx resource.OperatorContext, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { +func (r *Reconciler) recordIncompleteReconcileOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { if err := r.lastOpErrRecorder.RecordError(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { - ctx.Logger.Error(err, "failed to record last operation and last errors for etcd reconcilation") + ctx.Logger.Error(err, "failed to record last operation and last errors for etcd reconciliation") return ctrlutils.ReconcileWithError(err) } return exitReconcileStepResult diff --git a/internal/controller/etcd/reconcile_status.go b/internal/controller/etcd/reconcile_status.go new file mode 100644 index 000000000..9ead22979 --- /dev/null +++ b/internal/controller/etcd/reconcile_status.go @@ -0,0 +1,73 @@ +package etcd + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/health/status" + "github.com/gardener/etcd-druid/internal/operator/component" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/go-logr/logr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// mutateEtcdFn is a function which mutates the status of the passed etcd object +type mutateEtcdFn func(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, logger logr.Logger) ctrlutils.ReconcileStepResult + +func (r *Reconciler) reconcileStatus(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) + originalEtcd := etcd.DeepCopy() + mutateETCDStepFns := []mutateEtcdFn{ + r.mutateETCDStatusWithMemberStatusAndConditions, + r.inspectStatefulSetAndMutateETCDStatus, + } + for _, fn := range mutateETCDStepFns { + if stepResult := fn(ctx, etcd, sLog); ctrlutils.ShortCircuitReconcileFlow(stepResult) { + return stepResult + } + } + if err := r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)); err != nil { + sLog.Error(err, "failed to update etcd status") + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) mutateETCDStatusWithMemberStatusAndConditions(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, logger logr.Logger) ctrlutils.ReconcileStepResult { + statusCheck := status.NewChecker(r.client, r.config.EtcdMember.NotReadyThreshold, r.config.EtcdMember.UnknownThreshold) + if err := statusCheck.Check(ctx, logger, etcd); err != nil { + logger.Error(err, "Error executing status checks to update member status and conditions") + return ctrlutils.ReconcileWithError(err) + } + return ctrlutils.ContinueReconcile() +} + +func (r *Reconciler) inspectStatefulSetAndMutateETCDStatus(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, logger logr.Logger) ctrlutils.ReconcileStepResult { + sts, err := utils.GetStatefulSet(ctx, r.client, etcd) + if err != nil { + return ctrlutils.ReconcileWithError(err) + } + if sts != nil { + etcd.Status.Etcd = &druidv1alpha1.CrossVersionObjectReference{ + APIVersion: sts.APIVersion, + Kind: sts.Kind, + Name: sts.Name, + } + ready, _ := utils.IsStatefulSetReady(etcd.Spec.Replicas, sts) + etcd.Status.CurrentReplicas = sts.Status.CurrentReplicas + etcd.Status.ReadyReplicas = sts.Status.ReadyReplicas + etcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas + etcd.Status.Replicas = sts.Status.CurrentReplicas + etcd.Status.Ready = &ready + } else { + etcd.Status.CurrentReplicas = 0 + etcd.Status.ReadyReplicas = 0 + etcd.Status.UpdatedReplicas = 0 + etcd.Status.Ready = pointer.Bool(false) + } + return ctrlutils.ContinueReconcile() +} diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 128c2ce69..ef9535d55 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -18,28 +18,24 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/health/status" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/clientservice" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/operator/configmap" "github.com/gardener/etcd-druid/internal/operator/memberlease" "github.com/gardener/etcd-druid/internal/operator/peerservice" "github.com/gardener/etcd-druid/internal/operator/poddistruptionbudget" - "github.com/gardener/etcd-druid/internal/operator/resource" "github.com/gardener/etcd-druid/internal/operator/role" "github.com/gardener/etcd-druid/internal/operator/rolebinding" "github.com/gardener/etcd-druid/internal/operator/serviceaccount" "github.com/gardener/etcd-druid/internal/operator/snapshotlease" "github.com/gardener/etcd-druid/internal/operator/statefulset" - pkgutils "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" - "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -59,24 +55,28 @@ type Reconciler struct { // NewReconciler creates a new reconciler for Etcd. func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { imageVector, err := ctrlutils.CreateImageVector() - logger := log.Log.WithName(controllerName) if err != nil { return nil, err } - operatorReg := createAndInitializeOperatorRegistry(mgr.GetClient(), logger, config, imageVector) + return NewReconcilerWithImageVector(mgr, config, imageVector) +} + +func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, iv imagevector.ImageVector) (*Reconciler, error) { + logger := log.Log.WithName(controllerName) + operatorReg := createAndInitializeOperatorRegistry(mgr.GetClient(), config, iv) lastOpErrRecorder := ctrlutils.NewLastOperationErrorRecorder(mgr.GetClient(), logger) return &Reconciler{ client: mgr.GetClient(), config: config, recorder: mgr.GetEventRecorderFor(controllerName), - imageVector: imageVector, + imageVector: iv, logger: logger, operatorRegistry: operatorReg, lastOpErrRecorder: lastOpErrRecorder, }, nil } -type reconcileFn func(ctx resource.OperatorContext, objectKey client.ObjectKey) ctrlutils.ReconcileStepResult +type reconcileFn func(ctx component.OperatorContext, objectKey client.ObjectKey) ctrlutils.ReconcileStepResult // TODO: where/how is this being used? // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch @@ -91,12 +91,12 @@ type reconcileFn func(ctx resource.OperatorContext, objectKey client.ObjectKey) // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=events,verbs=create;get;list -// Reconcile manages the reconciliation of the Etcd resource to align it with its desired specifications. +// Reconcile manages the reconciliation of the Etcd component to align it with its desired specifications. // // The reconciliation process involves the following steps: -// 1. Deletion Handling: If the Etcd resource has a deletionTimestamp, initiate the deletion workflow. On error, requeue the request. +// 1. Deletion Handling: If the Etcd component has a deletionTimestamp, initiate the deletion workflow. On error, requeue the request. // 2. Spec Reconciliation : Determine whether the Etcd spec should be reconciled based on annotations and flags and if there is a need then reconcile spec. -// 3. Status Reconciliation: Always update the status of the Etcd resource to reflect its current state. +// 3. Status Reconciliation: Always update the status of the Etcd component to reflect its current state. // 4. Scheduled Requeue: Requeue the reconciliation request after a defined period (EtcdStatusSyncPeriod) to maintain sync. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { etcd := &druidv1alpha1.Etcd{} @@ -110,17 +110,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } // Execute the reconcile functions. runID := uuid.New().String() - operatorCtx := resource.NewOperatorContext(ctx, r.logger, runID) + operatorCtx := component.NewOperatorContext(ctx, r.logger, runID) for _, fn := range reconcileFns { if result := fn(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { return result.ReconcileResult() } } - return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult() } -func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logger, config *Config, imageVector imagevector.ImageVector) operator.Registry { +func (r *Reconciler) GetOperatorRegistry() operator.Registry { + return r.operatorRegistry +} + +func createAndInitializeOperatorRegistry(client client.Client, config *Config, imageVector imagevector.ImageVector) operator.Registry { reg := operator.NewRegistry() reg.Register(operator.ConfigMapKind, configmap.New(client)) reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, config.DisableEtcdServiceAccountAutomount)) @@ -135,7 +138,7 @@ func createAndInitializeOperatorRegistry(client client.Client, logger logr.Logge return reg } -func (r *Reconciler) reconcileEtcdDeletion(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) reconcileEtcdDeletion(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -148,7 +151,7 @@ func (r *Reconciler) reconcileEtcdDeletion(ctx resource.OperatorContext, etcdObj return ctrlutils.ContinueReconcile() } -func (r *Reconciler) reconcileSpec(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { +func (r *Reconciler) reconcileSpec(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result @@ -161,54 +164,6 @@ func (r *Reconciler) reconcileSpec(ctx resource.OperatorContext, etcdObjectKey c return ctrlutils.ContinueReconcile() } -func (r *Reconciler) reconcileStatus(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result - } - originalEtcd := etcd.DeepCopy() - sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) - sLog.Info("Started etcd status update") - - statusCheck := status.NewChecker(r.client, r.config.EtcdMember.NotReadyThreshold, r.config.EtcdMember.UnknownThreshold) - if err := statusCheck.Check(ctx, sLog, etcd); err != nil { - sLog.Error(err, "Error executing status checks") - return utils.ReconcileWithError(err) - } - - sts, err := pkgutils.GetStatefulSet(ctx, r.client, etcd) - if err != nil { - return utils.ReconcileWithError(err) - } - if sts != nil { - etcd.Status.Etcd = &druidv1alpha1.CrossVersionObjectReference{ - APIVersion: sts.APIVersion, - Kind: sts.Kind, - Name: sts.Name, - } - ready, _ := pkgutils.IsStatefulSetReady(etcd.Spec.Replicas, sts) - etcd.Status.CurrentReplicas = sts.Status.CurrentReplicas - etcd.Status.ReadyReplicas = sts.Status.ReadyReplicas - etcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas - etcd.Status.Replicas = sts.Status.CurrentReplicas - etcd.Status.Ready = &ready - } else { - etcd.Status.CurrentReplicas = 0 - etcd.Status.ReadyReplicas = 0 - etcd.Status.UpdatedReplicas = 0 - etcd.Status.Ready = pointer.Bool(false) - } - - err = r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)) - if err != nil { - sLog.Error(err, "Failed to update etcd status") - return utils.ReconcileWithError(err) - } - - sLog.Info("Finished etcd status update") - return utils.ContinueReconcile() -} - func (r *Reconciler) getLatestEtcd(ctx context.Context, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ctrlutils.ReconcileStepResult { if err := r.client.Get(ctx, objectKey, etcd); err != nil { if apierrors.IsNotFound(err) { diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index db159e0bf..096b42394 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -5,7 +5,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -13,9 +13,9 @@ import ( // LastOperationErrorRecorder records etcd.Status.LastOperation and etcd.Status.LastErrors type LastOperationErrorRecorder interface { - RecordStart(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error - RecordSuccess(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error - RecordError(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error + RecordStart(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error + RecordSuccess(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error + RecordError(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error } func NewLastOperationErrorRecorder(client client.Client, logger logr.Logger) LastOperationErrorRecorder { @@ -30,7 +30,7 @@ type lastOpErrRecorder struct { logger logr.Logger } -func (l *lastOpErrRecorder) RecordStart(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error { +func (l *lastOpErrRecorder) RecordStart(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error { const ( etcdReconcileStarted string = "Etcd cluster reconciliation is in progress" etcdDeletionStarted string = "Etcd cluster deletion is in progress" @@ -45,7 +45,7 @@ func (l *lastOpErrRecorder) RecordStart(ctx resource.OperatorContext, etcdObject return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateProcessing, description) } -func (l *lastOpErrRecorder) RecordSuccess(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error { +func (l *lastOpErrRecorder) RecordSuccess(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error { const ( etcdReconciledSuccessfully string = "Etcd cluster has been successfully reconciled" etcdDeletedSuccessfully string = "Etcd cluster has been successfully deleted" @@ -61,13 +61,13 @@ func (l *lastOpErrRecorder) RecordSuccess(ctx resource.OperatorContext, etcdObje return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateSucceeded, description) } -func (l *lastOpErrRecorder) RecordError(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { +func (l *lastOpErrRecorder) RecordError(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { description += " Operation will be retried." lastErrors := druiderr.MapToLastErrors(errs) return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateError, description, lastErrors...) } -func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx resource.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, operationState druidv1alpha1.LastOperationState, description string, lastErrors ...druidv1alpha1.LastError) error { +func (l *lastOpErrRecorder) recordLastOperationAndErrors(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, operationState druidv1alpha1.LastOperationState, description string, lastErrors ...druidv1alpha1.LastError) error { etcd := &druidv1alpha1.Etcd{} if err := l.client.Get(ctx, etcdObjectKey, etcd); err != nil { return err diff --git a/internal/errors/errors.go b/internal/errors/errors.go index ecdb77a76..76f18567a 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -48,9 +48,10 @@ func MapToLastErrors(errs []error) []druidv1alpha1.LastError { for _, err := range errs { druidErr := &DruidError{} if errors.As(err, &druidErr) { + desc := fmt.Sprintf("[Operation: %s, Code: %s] message: %s, cause: %s", druidErr.Operation, druidErr.Code, druidErr.Message, druidErr.Cause.Error()) lastErr := druidv1alpha1.LastError{ Code: druidErr.Code, - Description: druidErr.Message, + Description: desc, LastUpdateTime: metav1.NewTime(time.Now().UTC()), } lastErrs = append(lastErrs, lastErr) diff --git a/internal/health/condition/builder.go b/internal/health/condition/builder.go index a26d6cb70..ca796c867 100644 --- a/internal/health/condition/builder.go +++ b/internal/health/condition/builder.go @@ -111,7 +111,7 @@ func (b *defaultBuilder) Build(replicas int32) []druidv1alpha1.Condition { if condition.Status == "" { condition.Status = druidv1alpha1.ConditionUnknown } - condition.Reason = ConditionNotChecked + condition.Reason = NotChecked condition.Message = "etcd cluster has been scaled down" } else { condition.Status = res.Status() diff --git a/internal/health/condition/check_backup_ready.go b/internal/health/condition/check_backup_ready.go index 077adaea6..4551faa9a 100644 --- a/internal/health/condition/check_backup_ready.go +++ b/internal/health/condition/check_backup_ready.go @@ -37,8 +37,8 @@ const ( BackupFailed string = "BackupFailed" // Unknown is a constant that means that the etcd backup status is currently not known Unknown string = "Unknown" - // ConditionNotChecked is a constant that means that the etcd backup status has not been updated or rechecked - ConditionNotChecked string = "ConditionNotChecked" + // NotChecked is a constant that means that the etcd backup status has not been updated or rechecked + NotChecked string = "NotChecked" ) func (a *backupReadyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) Result { diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go index a72a7a89c..18cb2fcd9 100644 --- a/internal/health/condition/check_data_volumes_ready.go +++ b/internal/health/condition/check_data_volumes_ready.go @@ -49,7 +49,7 @@ func (d *dataVolumesReady) Check(ctx context.Context, etcd druidv1alpha1.Etcd) R return res } - pvcEvents, err := utils.FetchPVCWarningEventsForStatefulSet(ctx, d.cl, sts) + pvcEvents, err := utils.FetchPVCWarningMessageForStatefulSet(ctx, d.cl, sts) if err != nil { res.reason = "UnableToFetchWarningEventsForDataVolumes" res.message = fmt.Sprintf("Unable to fetch warning events for PVCs used by StatefulSet %v: %s", kutil.Key(sts.Name, sts.Namespace), err.Error()) diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 102d44aac..3b44c8490 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -33,7 +33,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) svc := &corev1.Service{} svcObjectKey := getObjectKey(etcd) @@ -46,11 +46,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting client service: %v for etcd: %v", svcObjectKey, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, svc.Name) + if metav1.IsControlledBy(svc, etcd) { + resourceNames = append(resourceNames, svc.Name) + } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) svc := emptyClientService(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { @@ -64,11 +66,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error during create or update of client service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("synced", "resource", "client-service", "objectKey", objectKey, "result", result) + ctx.Logger.Info("synced", "component", "client-service", "objectKey", objectKey, "result", result) return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of client service", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyClientService(objectKey)); err != nil { @@ -83,11 +85,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "Failed to delete client service", ) } - ctx.Logger.Info("deleted", "resource", "client-service", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "client-service", "objectKey", objectKey) return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 751cbd981..af1b85a0f 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -23,7 +23,7 @@ import ( . "github.com/onsi/gomega/gstruct" "k8s.io/utils/pointer" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -79,7 +79,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newClientService(etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -127,7 +127,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() etcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) latestClientSvc, getErr := getLatestClientService(cl, etcd) if tc.expectedErr != nil { @@ -183,7 +183,7 @@ func TestSyncWhenServiceExists(t *testing.T) { Build() // ********************* test sync with updated ports ********************* operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) syncErr := operator.Sync(opCtx, updatedEtcd) latestClientSvc, getErr := getLatestClientService(cl, updatedEtcd) @@ -236,7 +236,7 @@ func TestTriggerDelete(t *testing.T) { // ********************* Setup ********************* cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) if tc.svcExists { syncErr := operator.Sync(opCtx, etcd) g.Expect(syncErr).ToNot(HaveOccurred()) diff --git a/internal/operator/resource/types.go b/internal/operator/component/types.go similarity index 91% rename from internal/operator/resource/types.go rename to internal/operator/component/types.go index 3f1dfbe98..7563b2ec5 100644 --- a/internal/operator/resource/types.go +++ b/internal/operator/component/types.go @@ -1,4 +1,4 @@ -package resource +package component import ( "context" @@ -38,8 +38,8 @@ type Operator interface { GetExistingResourceNames(ctx OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) // TriggerDelete triggers the deletion of all resources that this Operator manages. TriggerDelete(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error - // Sync synchronizes all resources that this Operator manages. If a resource does not exist then it will + // Sync synchronizes all resources that this Operator manages. If a component does not exist then it will // create it. If there are changes in the owning Etcd resource that transpires changes to one or more resources - // managed by this Operator then those resource(s) will be either be updated or a deletion is triggered. + // managed by this Operator then those component(s) will be either be updated or a deletion is triggered. Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error } diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 00764fde5..37c96fb06 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -7,7 +7,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils" @@ -30,7 +30,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objKey := getObjectKey(etcd) cm := &corev1.ConfigMap{} @@ -43,11 +43,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting ConfigMap: %v for etcd: %v", objKey, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, cm.Name) + if metav1.IsControlledBy(cm, etcd) { + resourceNames = append(resourceNames, cm.Name) + } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { cm := emptyConfigMap(getObjectKey(etcd)) result, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { return buildResource(etcd, cm) @@ -66,11 +68,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error when computing CheckSum for configmap for etcd: %v", etcd.GetNamespaceName())) } ctx.Data[common.ConfigMapCheckSumKey] = checkSum - ctx.Logger.Info("synced", "resource", "configmap", "name", cm.Name, "result", result) + ctx.Logger.Info("synced", "component", "configmap", "name", cm.Name, "result", result) return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyConfigMap(objectKey)); err != nil { @@ -85,11 +87,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "Failed to delete configmap", ) } - ctx.Logger.Info("deleted", "resource", "configmap", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "configmap", "objectKey", objectKey) return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 779aec717..928c60207 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -9,7 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -64,7 +64,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newConfigMap(g, etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) cmNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -79,16 +79,31 @@ func TestGetExistingResourceNames(t *testing.T) { // ----------------------------------- Sync ----------------------------------- func TestSyncWhenNoConfigMapExists(t *testing.T) { testCases := []struct { - name string - createErr *apierrors.StatusError - expectedErr *druiderr.DruidError + name string + etcdReplicas int32 + createErr *apierrors.StatusError + clientTLSEnabled bool + peerTLSEnabled bool + expectedErr *druiderr.DruidError }{ { - name: "should create when no configmap exists", + name: "should create when no configmap exists for single node etcd cluster", + clientTLSEnabled: true, + peerTLSEnabled: false, + etcdReplicas: 1, + }, + { + name: "should create when no configmap exists for multi-node etcd cluster", + clientTLSEnabled: true, + peerTLSEnabled: true, + etcdReplicas: 3, }, { - name: "return error when create client request fails", - createErr: testutils.TestAPIInternalErr, + name: "return error when create client request fails", + etcdReplicas: 3, + clientTLSEnabled: true, + peerTLSEnabled: true, + createErr: testutils.TestAPIInternalErr, expectedErr: &druiderr.DruidError{ Code: ErrSyncConfigMap, Cause: testutils.TestAPIInternalErr, @@ -98,12 +113,12 @@ func TestSyncWhenNoConfigMapExists(t *testing.T) { } g := NewWithT(t) t.Parallel() - etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().WithPeerTLS().Build() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + etcd := buildEtcd(tc.etcdReplicas, tc.clientTLSEnabled, tc.peerTLSEnabled) cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) latestConfigMap, getErr := getLatestConfigMap(cl, etcd) if tc.expectedErr != nil { @@ -118,6 +133,59 @@ func TestSyncWhenNoConfigMapExists(t *testing.T) { } } +func TestPrepareInitialCluster(t *testing.T) { + testCases := []struct { + name string + peerTLSEnabled bool + etcdReplicas int32 + etcdSpecServerPort *int32 + expectedInitialCluster string + }{ + { + name: "should create initial cluster for single node etcd cluster when peer TLS is enabled", + etcdReplicas: 1, + peerTLSEnabled: true, + etcdSpecServerPort: pointer.Int32(2222), + expectedInitialCluster: "etcd-test-0=https://etcd-test-0.etcd-test-peer.test-ns.svc:2222", + }, + { + name: "should create initial cluster for single node etcd cluster when peer TLS is disabled", + etcdReplicas: 1, + peerTLSEnabled: false, + expectedInitialCluster: "etcd-test-0=http://etcd-test-0.etcd-test-peer.test-ns.svc:2380", + }, + { + name: "should create initial cluster for multi node etcd cluster when peer TLS is enabled", + etcdReplicas: 3, + peerTLSEnabled: true, + etcdSpecServerPort: pointer.Int32(2333), + expectedInitialCluster: "etcd-test-0=https://etcd-test-0.etcd-test-peer.test-ns.svc:2333,etcd-test-1=https://etcd-test-1.etcd-test-peer.test-ns.svc:2333,etcd-test-2=https://etcd-test-2.etcd-test-peer.test-ns.svc:2333", + }, + } + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + etcd := buildEtcd(tc.etcdReplicas, true, tc.peerTLSEnabled) + etcd.Spec.Etcd.ServerPort = tc.etcdSpecServerPort + peerScheme := utils.IfConditionOr[string](etcd.Spec.Etcd.PeerUrlTLS != nil, "https", "http") + actualInitialCluster := prepareInitialCluster(etcd, peerScheme) + g.Expect(actualInitialCluster).To(Equal(tc.expectedInitialCluster)) + }) + } +} + +func buildEtcd(replicas int32, clientTLSEnabled, peerTLSEnabled bool) *druidv1alpha1.Etcd { + etcdBuilder := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(replicas) + if clientTLSEnabled { + etcdBuilder.WithClientTLS() + } + if peerTLSEnabled { + etcdBuilder.WithPeerTLS() + } + return etcdBuilder.Build() +} + func TestSyncWhenConfigMapExists(t *testing.T) { testCases := []struct { name string @@ -149,7 +217,7 @@ func TestSyncWhenConfigMapExists(t *testing.T) { updatedEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().WithPeerTLS().Build() updatedEtcd.UID = originalEtcd.UID operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, updatedEtcd) latestConfigMap, getErr := getLatestConfigMap(cl, updatedEtcd) if tc.expectedErr != nil { @@ -201,7 +269,7 @@ func TestTriggerDelete(t *testing.T) { // ********************* Setup ********************* cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) if tc.cmExists { syncErr := operator.Sync(opCtx, etcd) g.Expect(syncErr).ToNot(HaveOccurred()) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 51cd690e3..592b42ae9 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/hashicorp/go-multierror" @@ -26,7 +26,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) leaseList := &coordinationv1.LeaseList{} err := r.client.List(ctx, @@ -40,12 +40,14 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * fmt.Sprintf("Error listing member leases for etcd: %v", etcd.GetNamespaceName())) } for _, lease := range leaseList.Items { - resourceNames = append(resourceNames, lease.Name) + if metav1.IsControlledBy(&lease, etcd) { + resourceNames = append(resourceNames, lease.Name) + } } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKeys := getObjectKeys(etcd) createTasks := make([]utils.OperatorTask, len(objectKeys)) var errs error @@ -54,7 +56,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) objKey := objKey // capture the range variable createTasks[i] = utils.OperatorTask{ Name: "CreateOrUpdate-" + objKey.String(), - Fn: func(ctx resource.OperatorContext) error { + Fn: func(ctx component.OperatorContext) error { return r.doCreateOrUpdate(ctx, etcd, objKey) }, } @@ -67,7 +69,7 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) return errs } -func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { +func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, objKey client.ObjectKey) error { lease := emptyMemberLease(objKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { buildResource(etcd, lease) @@ -83,7 +85,7 @@ func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1a return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of member leases") if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, @@ -94,11 +96,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "TriggerDelete", fmt.Sprintf("Failed to delete member leases for etcd: %v", etcd.GetNamespaceName())) } - ctx.Logger.Info("deleted", "resource", "member-leases") + ctx.Logger.Info("deleted", "component", "member-leases") return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index 6aed79008..d669cc7ff 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -9,7 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -89,7 +89,7 @@ func TestGetExistingResourceNames(t *testing.T) { } } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) memberLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -175,7 +175,7 @@ func TestSync(t *testing.T) { } // ***************** Setup operator and test ***************** operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, updatedEtcd) memberLeasesPostSync := getLatestMemberLeases(g, cl, updatedEtcd) if tc.expectedErr != nil { @@ -252,7 +252,7 @@ func TestTriggerDelete(t *testing.T) { cl := fakeClientBuilder.Build() // ***************** Setup operator and test ***************** operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) memberLeasesBeforeDelete := getLatestMemberLeases(g, cl, etcd) fmt.Println(memberLeasesBeforeDelete) err := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index a1a3417be..909cbc332 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -28,7 +28,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) svcObjectKey := getObjectKey(etcd) svc := &corev1.Service{} @@ -41,11 +41,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting peer service: %s for etcd: %v", svcObjectKey.Name, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, svc.Name) + if metav1.IsControlledBy(svc, etcd) { + resourceNames = append(resourceNames, svc.Name) + } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) svc := emptyPeerService(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { @@ -59,11 +61,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error during create or update of peer service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("synced", "resource", "peer-service", "objectKey", objectKey, "result", result) + ctx.Logger.Info("synced", "component", "peer-service", "objectKey", objectKey, "result", result) return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of peer service") err := r.client.Delete(ctx, emptyPeerService(objectKey)) @@ -79,11 +81,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph fmt.Sprintf("Failed to delete peer service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("deleted", "resource", "peer-service", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "peer-service", "objectKey", objectKey) return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index ead7378c4..7c5aefda7 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -21,7 +21,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -78,7 +78,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newPeerService(etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -126,7 +126,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { } etcd := etcdBuilder.Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) latestPeerService, getErr := getLatestPeerService(cl, etcd) if tc.expectedError != nil { @@ -174,7 +174,7 @@ func TestSyncWhenServiceExists(t *testing.T) { WithObjects(newPeerService(existingEtcd)). Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithEtcdServerPort(tc.updateWithPort).Build() syncErr := operator.Sync(opCtx, updatedEtcd) latestPeerService, getErr := getLatestPeerService(cl, updatedEtcd) @@ -230,7 +230,7 @@ func TestPeerServiceTriggerDelete(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.TriggerDelete(opCtx, etcd) _, getErr := getLatestPeerService(cl, etcd) if tc.expectError != nil { diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 8fbd92101..29a803d29 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" policyv1 "k8s.io/api/policy/v1" @@ -26,7 +26,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) pdb := &policyv1.PodDisruptionBudget{} @@ -39,11 +39,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, pdb.Name) + if metav1.IsControlledBy(pdb, etcd) { + resourceNames = append(resourceNames, pdb.Name) + } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) pdb := emptyPodDisruptionBudget(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { @@ -57,11 +59,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error during create or update of PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("synced", "resource", "pod-disruption-budget", "objectKey", objectKey, "result", result) + ctx.Logger.Info("synced", "component", "pod-disruption-budget", "objectKey", objectKey, "result", result) return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of PDB") pdbObjectKey := getObjectKey(etcd) if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPodDisruptionBudget(pdbObjectKey))); err != nil { @@ -70,11 +72,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "TriggerDelete", fmt.Sprintf("Failed to delete PDB: %v for etcd: %v", pdbObjectKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("deleted", "resource", "pod-disruption-budget", "objectKey", pdbObjectKey) + ctx.Logger.Info("deleted", "component", "pod-disruption-budget", "objectKey", pdbObjectKey) return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index f5946b072..8a54b9880 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -63,7 +63,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newPodDisruptionBudget(etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) pdbNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -113,7 +113,7 @@ func TestSyncWhenNoPDBExists(t *testing.T) { etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) latestPDB, getErr := getLatestPodDisruptionBudget(cl, etcd) if tc.expectedErr != nil { @@ -169,7 +169,7 @@ func TestSyncWhenPDBExists(t *testing.T) { WithObjects(newPodDisruptionBudget(existingEtcd)). Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithReplicas(tc.updatedEtcdReplicas).Build() syncErr := operator.Sync(opCtx, updatedEtcd) latestPDB, getErr := getLatestPodDisruptionBudget(cl, updatedEtcd) @@ -223,7 +223,7 @@ func TestTriggerDelete(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.TriggerDelete(opCtx, etcd) _, getErr := getLatestPodDisruptionBudget(cl, etcd) if tc.expectedErr != nil { diff --git a/internal/operator/registry.go b/internal/operator/registry.go index 4d2dbf733..18858595f 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -1,105 +1,65 @@ package operator import ( - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" ) -// Registry is a facade which gives access to all resource operators. +// Registry is a facade which gives access to all component operators. type Registry interface { - // Register provides consumers to register an operator against the kind of resource it operates on. - Register(kind Kind, operator resource.Operator) - // AllOperators gives a map, where the key is the Kind of resource that an operator manages and the value is an Operator itself. - AllOperators() map[Kind]resource.Operator + // Register provides consumers to register an operator against the kind of component it operates on. + Register(kind Kind, operator component.Operator) + // AllOperators gives a map, where the key is the Kind of component that an operator manages and the value is an Operator itself. + AllOperators() map[Kind]component.Operator // GetOperator gets an operator that operates on the kind. // Returns the operator if an operator is found, else nil will be returned. - GetOperator(kind Kind) resource.Operator + GetOperator(kind Kind) component.Operator } type Kind string const ( - // StatefulSetKind indicates that the kind of resource is a StatefulSet. + // StatefulSetKind indicates that the kind of component is a StatefulSet. StatefulSetKind Kind = "StatefulSet" - // ServiceAccountKind indicates that the kind of resource is a ServiceAccount. + // ServiceAccountKind indicates that the kind of component is a ServiceAccount. ServiceAccountKind Kind = "ServiceAccount" - // RoleKind indicates that the kind of resource is a Role. + // RoleKind indicates that the kind of component is a Role. RoleKind Kind = "Role" - // RoleBindingKind indicates that the kind of resource is RoleBinding + // RoleBindingKind indicates that the kind of component is RoleBinding RoleBindingKind Kind = "RoleBinding" - // MemberLeaseKind indicates that the kind of resource is a Lease used for an etcd member heartbeat. + // MemberLeaseKind indicates that the kind of component is a Lease used for an etcd member heartbeat. MemberLeaseKind Kind = "MemberLease" - // SnapshotLeaseKind indicates that the kind of resource is a Lease used to capture snapshot information. + // SnapshotLeaseKind indicates that the kind of component is a Lease used to capture snapshot information. SnapshotLeaseKind Kind = "SnapshotLease" - // ConfigMapKind indicates that the kind of resource is a ConfigMap. + // ConfigMapKind indicates that the kind of component is a ConfigMap. ConfigMapKind Kind = "ConfigMap" - // PeerServiceKind indicates that the kind of resource is a Service used for etcd peer communication. + // PeerServiceKind indicates that the kind of component is a Service used for etcd peer communication. PeerServiceKind Kind = "PeerService" - // ClientServiceKind indicates that the kind of resource is a Service used for etcd client communication. + // ClientServiceKind indicates that the kind of component is a Service used for etcd client communication. ClientServiceKind Kind = "ClientService" - // PodDisruptionBudgetKind indicates that the kind of resource is a PodDisruptionBudget. + // PodDisruptionBudgetKind indicates that the kind of component is a PodDisruptionBudget. PodDisruptionBudgetKind Kind = "PodDisruptionBudget" ) type registry struct { - operators map[Kind]resource.Operator + operators map[Kind]component.Operator } // NewRegistry creates a new instance of a Registry. func NewRegistry() Registry { - operators := make(map[Kind]resource.Operator) + operators := make(map[Kind]component.Operator) return registry{ operators: operators, } } -func (r registry) Register(kind Kind, operator resource.Operator) { +func (r registry) Register(kind Kind, operator component.Operator) { r.operators[kind] = operator } -func (r registry) GetOperator(kind Kind) resource.Operator { +func (r registry) GetOperator(kind Kind) component.Operator { return r.operators[kind] } -func (r registry) AllOperators() map[Kind]resource.Operator { +func (r registry) AllOperators() map[Kind]component.Operator { return r.operators -} - -func (r registry) StatefulSetOperator() resource.Operator { - return r.operators[StatefulSetKind] -} - -func (r registry) ServiceAccountOperator() resource.Operator { - return r.operators[ServiceAccountKind] -} - -func (r registry) RoleOperator() resource.Operator { - return r.operators[RoleKind] -} - -func (r registry) RoleBindingOperator() resource.Operator { - return r.operators[RoleBindingKind] -} - -func (r registry) MemberLeaseOperator() resource.Operator { - return r.operators[MemberLeaseKind] -} - -func (r registry) SnapshotLeaseOperator() resource.Operator { - return r.operators[SnapshotLeaseKind] -} - -func (r registry) ConfigMapOperator() resource.Operator { - return r.operators[ConfigMapKind] -} - -func (r registry) PeerServiceOperator() resource.Operator { - return r.operators[PeerServiceKind] -} - -func (r registry) ClientServiceOperator() resource.Operator { - return r.operators[ClientServiceKind] -} - -func (r registry) PodDisruptionBudgetOperator() resource.Operator { - return r.operators[PodDisruptionBudgetKind] -} +} \ No newline at end of file diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 3ab02ce61..2365e44d6 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -2,11 +2,12 @@ package role import ( "fmt" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" @@ -25,7 +26,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) role := &rbacv1.Role{} @@ -38,11 +39,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting role: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, role.Name) + if metav1.IsControlledBy(role, etcd) { + resourceNames = append(resourceNames, role.Name) + } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) role := emptyRole(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { @@ -56,11 +59,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error during create or update of role %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("synced", "resource", "role", "objectKey", objectKey, "result", result) + ctx.Logger.Info("synced", "component", "role", "objectKey", objectKey, "result", result) return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyRole(objectKey)); err != nil { @@ -74,11 +77,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph fmt.Sprintf("Failed to delete role: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("deleted", "resource", "role", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "role", "objectKey", objectKey) return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } @@ -122,7 +125,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.RoleComponentName, - druidv1alpha1.LabelAppNameKey: etcd.GetRoleName(), + druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleName(), ":", "-"), // role name contains `:` which is not an allowed character as a label value. } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) } diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 15236725e..2898f0606 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -64,7 +64,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newRole(etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) roleNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -107,7 +107,7 @@ func TestSync(t *testing.T) { WithCreateError(tc.createErr). Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) latestRole, getErr := getLatestRole(cl, etcd) if tc.expectedErr != nil { @@ -166,7 +166,7 @@ func TestTriggerDelete(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) deleteErr := operator.TriggerDelete(opCtx, etcd) latestRole, getErr := getLatestRole(cl, etcd) if tc.expectedErr != nil { diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index fcdc4ce8a..3b0ea7c08 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -2,11 +2,12 @@ package rolebinding import ( "fmt" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" @@ -25,7 +26,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) rb := &rbacv1.RoleBinding{} @@ -38,11 +39,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, rb.Name) + if metav1.IsControlledBy(rb, etcd) { + resourceNames = append(resourceNames, rb.Name) + } return resourceNames, nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) err := r.client.Delete(ctx, emptyRoleBinding(objectKey)) @@ -57,11 +60,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph fmt.Sprintf("Failed to delete role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("deleted", "resource", "role-binding", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "role-binding", "objectKey", objectKey) return nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) rb := emptyRoleBinding(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { @@ -75,11 +78,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error during create or update of role-binding %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("synced", "resource", "role", "objectKey", objectKey, "result", result) + ctx.Logger.Info("synced", "component", "role", "objectKey", objectKey, "result", result) return nil } -func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { +func (r _resource) Exists(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { rb := &rbacv1.RoleBinding{} if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { if errors.IsNotFound(err) { @@ -90,7 +93,7 @@ func (r _resource) Exists(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd return true, nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } @@ -129,7 +132,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.RoleBindingComponentName, - druidv1alpha1.LabelAppNameKey: etcd.GetRoleBindingName(), + druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleBindingName(), ":", "-"), // role-binding name contains `:` which is not an allowed character as a label value. } return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) } diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index 2fd1e35d9..4637aa675 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -65,7 +65,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newRoleBinding(etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) roleBindingNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -106,7 +106,7 @@ func TestSync(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) latestRoleBinding, getErr := getLatestRoleBinding(cl, etcd) if tc.expectedErr != nil { @@ -165,7 +165,7 @@ func TestTriggerDelete(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index a6db8ea0f..91ca5fbf3 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -27,7 +27,7 @@ type _resource struct { disableAutoMount bool } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) sa := &corev1.ServiceAccount{} objectKey := getObjectKey(etcd) @@ -40,11 +40,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * "GetExistingResourceNames", fmt.Sprintf("Error getting service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - resourceNames = append(resourceNames, sa.Name) + if metav1.IsControlledBy(sa, etcd) { + resourceNames = append(resourceNames, sa.Name) + } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) sa := emptyServiceAccount(objectKey) opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, sa, func() error { @@ -58,11 +60,11 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) fmt.Sprintf("Error during create or update of service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), ) } - ctx.Logger.Info("synced", "resource", "service-account", "objectKey", objectKey, "result", opResult) + ctx.Logger.Info("synced", "component", "service-account", "objectKey", objectKey, "result", opResult) return nil } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of service account") objectKey := getObjectKey(etcd) if err := r.client.Delete(ctx, emptyServiceAccount(objectKey)); err != nil { @@ -75,11 +77,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "TriggerDelete", fmt.Sprintf("Failed to delete service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("deleted", "resource", "service-account", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "service-account", "objectKey", objectKey) return nil } -func New(client client.Client, disableAutomount bool) resource.Operator { +func New(client client.Client, disableAutomount bool) component.Operator { return &_resource{ client: client, disableAutoMount: disableAutomount, diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index e889ebfa3..f6a34fdec 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -6,7 +6,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" @@ -61,7 +61,7 @@ func TestGetExistingResourceNames(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl, true) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) saNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -112,7 +112,7 @@ func TestSync(t *testing.T) { cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() operator := New(cl, tc.disableAutoMount) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) latestSA, getErr := getLatestServiceAccount(cl, etcd) if tc.expectedErr != nil { @@ -166,7 +166,7 @@ func TestTriggerDelete(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl, false) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestSA, getErr := getLatestServiceAccount(cl, etcd) if tc.expectedErr != nil { diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 050ed0326..700ca5b80 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -2,17 +2,17 @@ package snapshotlease import ( "context" + "errors" "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/hashicorp/go-multierror" coordinationv1 "k8s.io/api/coordination/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -27,7 +27,7 @@ type _resource struct { client client.Client } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 2) // We have to get snapshot leases one lease at a time and cannot use label-selector based listing // because currently snapshot lease do not have proper labels on them. In this new code @@ -43,7 +43,7 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * Message: fmt.Sprintf("Error getting delta snapshot lease: %v for etcd: %v", deltaSnapshotObjectKey, etcd.GetNamespaceName()), } } - if deltaSnapshotLease != nil { + if deltaSnapshotLease != nil && metav1.IsControlledBy(deltaSnapshotLease, etcd) { resourceNames = append(resourceNames, deltaSnapshotLease.Name) } fullSnapshotObjectKey := client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace} @@ -56,13 +56,13 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * Message: fmt.Sprintf("Error getting full snapshot lease: %v for etcd: %v", fullSnapshotObjectKey, etcd.GetNamespaceName()), } } - if fullSnapshotLease != nil { + if fullSnapshotLease != nil && metav1.IsControlledBy(fullSnapshotLease, etcd) { resourceNames = append(resourceNames, fullSnapshotLease.Name) } return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { ctx.Logger.Info("Backup has been disabled. Triggering delete of snapshot leases") return r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { @@ -75,23 +75,20 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) objectKeys := getObjectKeys(etcd) syncTasks := make([]utils.OperatorTask, len(objectKeys)) - var errs error for i, objKey := range objectKeys { objKey := objKey // capture the range variable syncTasks[i] = utils.OperatorTask{ Name: "CreateOrUpdate-" + objKey.String(), - Fn: func(ctx resource.OperatorContext) error { + Fn: func(ctx component.OperatorContext) error { return r.doCreateOrUpdate(ctx, etcd, objKey) }, } } - - errs = multierror.Append(errs, utils.RunConcurrently(ctx, syncTasks)...) - return errs + return errors.Join(utils.RunConcurrently(ctx, syncTasks)...) } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of snapshot leases") if err := r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { return druiderr.WrapError(err, @@ -101,11 +98,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph }); err != nil { return err } - ctx.Logger.Info("deleted", "resource", "snapshot-leases") + ctx.Logger.Info("deleted", "component", "snapshot-leases") return nil } -func (r _resource) deleteAllSnapshotLeases(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, wrapErrFn func(error) error) error { +func (r _resource) deleteAllSnapshotLeases(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, wrapErrFn func(error) error) error { if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(etcd.Namespace), @@ -115,7 +112,7 @@ func (r _resource) deleteAllSnapshotLeases(ctx resource.OperatorContext, etcd *d return nil } -func New(client client.Client) resource.Operator { +func New(client client.Client) component.Operator { return &_resource{ client: client, } @@ -124,7 +121,7 @@ func New(client client.Client) resource.Operator { func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*coordinationv1.Lease, error) { lease := &coordinationv1.Lease{} if err := r.client.Get(ctx, objectKey, lease); err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return nil, nil } return nil, err @@ -132,7 +129,7 @@ func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*c return lease, nil } -func (r _resource) doCreateOrUpdate(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) error { +func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) error { lease := emptySnapshotLease(leaseObjectKey) opResult, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, lease, func() error { buildResource(etcd, lease) diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 67f784699..b46c70f32 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -8,7 +8,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -74,7 +74,7 @@ func TestGetExistingResourceNames(t *testing.T) { fakeClientBuilder.WithObjects(newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) } operator := New(fakeClientBuilder.Build()) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) actualSnapshotLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -114,7 +114,7 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) latestSnapshotLeases, listErr := getLatestSnapshotLeases(cl, etcd) if tc.expectedErr != nil { @@ -165,7 +165,7 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { newFullSnapshotLease(nonTargetEtcd)). Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, updatedEtcd) latestSnapshotLeases, listErr := getLatestSnapshotLeases(cl, updatedEtcd) g.Expect(listErr).ToNot(HaveOccurred()) @@ -232,7 +232,7 @@ func TestTriggerDelete(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestSnapshotLeases, snapshotLeaseListErr := getLatestSnapshotLeases(cl, etcd) if tc.expectedErr != nil { diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 9030743fc..9e1941d31 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -7,7 +7,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" druidutils "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -100,7 +100,7 @@ func newStsBuilder(client client.Client, }, nil } -func (b *stsBuilder) Build(ctx resource.OperatorContext) error { +func (b *stsBuilder) Build(ctx component.OperatorContext) error { b.createStatefulSetObjectMeta() if err := b.createStatefulSetSpec(ctx); err != nil { return err @@ -125,7 +125,7 @@ func (b *stsBuilder) getStatefulSetLabels() map[string]string { return utils.MergeMaps[string, string](b.etcd.GetDefaultLabels(), stsLabels) } -func (b *stsBuilder) createStatefulSetSpec(ctx resource.OperatorContext) error { +func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error { podVolumes, err := b.getPodVolumes(ctx) if err != nil { return err @@ -179,7 +179,7 @@ func (b *stsBuilder) getHostAliases() []corev1.HostAlias { } } -func (b *stsBuilder) getPodTemplateAnnotations(ctx resource.OperatorContext) map[string]string { +func (b *stsBuilder) getPodTemplateAnnotations(ctx component.OperatorContext) map[string]string { if configMapCheckSum, ok := ctx.Data[common.ConfigMapCheckSumKey]; ok { return utils.MergeMaps[string](b.etcd.Spec.Annotations, map[string]string{ common.ConfigMapCheckSumKey: configMapCheckSum, @@ -649,7 +649,7 @@ func (b *stsBuilder) getSecretVolumeMounts() []corev1.VolumeMount { } // getPodVolumes gets volumes that needs to be mounted onto the etcd StatefulSet pods -func (b *stsBuilder) getPodVolumes(ctx resource.OperatorContext) ([]corev1.Volume, error) { +func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volume, error) { volumes := []corev1.Volume{ { Name: "etcd-config-file", @@ -740,7 +740,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { } } -func (b *stsBuilder) getBackupVolume(ctx resource.OperatorContext) (*corev1.Volume, error) { +func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Volume, error) { if b.provider == nil { return nil, nil } diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 4d4aca6b9..bb3040d45 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -9,7 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/component-base/featuregate" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -31,7 +31,7 @@ type _resource struct { useEtcdWrapper bool } -func New(client client.Client, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) resource.Operator { +func New(client client.Client, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) component.Operator { return &_resource{ client: client, imageVector: imageVector, @@ -39,7 +39,8 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates } } -func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { + resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) sts, err := r.getExistingStatefulSet(ctx, objectKey) if err != nil { @@ -49,12 +50,15 @@ func (r _resource) GetExistingResourceNames(ctx resource.OperatorContext, etcd * fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } if sts == nil { - return []string{}, nil + return resourceNames, nil } - return []string{sts.Name}, nil + if metav1.IsControlledBy(sts, etcd) { + resourceNames = append(resourceNames, sts.Name) + } + return resourceNames, nil } -func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { var ( existingSTS *appsv1.StatefulSet err error @@ -73,16 +77,16 @@ func (r _resource) Sync(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) } // StatefulSet exists, check if TLS has been enabled for peer communication, if yes then it is currently a multistep // process to ensure that all members are updated and establish peer TLS communication. - if err = r.handlePeerTLSEnabled(ctx, etcd, existingSTS); err != nil { - return fmt.Errorf("error handling peer TLS: %w", err) - } - if err = r.handleImmutableFieldUpdates(ctx, etcd, existingSTS); err != nil { - return fmt.Errorf("error handling immutable field updates: %w", err) + if err = r.handlePeerTLSChanges(ctx, etcd, existingSTS); err != nil { + return druiderr.WrapError(err, + ErrSyncStatefulSet, + "Sync", + fmt.Sprintf("Error while handling peer URL TLS change for StatefulSet: %v, etcd: %v", objectKey, etcd.GetNamespaceName())) } return r.createOrPatch(ctx, etcd) } -func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of StatefulSet", "objectKey", objectKey) err := r.client.Delete(ctx, emptyStatefulSet(objectKey)) @@ -96,11 +100,11 @@ func (r _resource) TriggerDelete(ctx resource.OperatorContext, etcd *druidv1alph "TriggerDelete", fmt.Sprintf("Failed to delete StatefulSet: %v for etcd %v", objectKey, etcd.GetNamespaceName())) } - ctx.Logger.Info("deleted", "resource", "statefulset", "objectKey", objectKey) + ctx.Logger.Info("deleted", "component", "statefulset", "objectKey", objectKey) return nil } -func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, objectKey client.ObjectKey) (*appsv1.StatefulSet, error) { +func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, objectKey client.ObjectKey) (*appsv1.StatefulSet, error) { sts := emptyStatefulSet(objectKey) if err := r.client.Get(ctx, objectKey, sts); err != nil { if errors.IsNotFound(err) { @@ -112,8 +116,8 @@ func (r _resource) getExistingStatefulSet(ctx resource.OperatorContext, objectKe } // createOrPatchWithReplicas ensures that the StatefulSet is updated with all changes from passed in etcd but the replicas set on the StatefulSet -// are taken from the passed in replicas and not from the etcd resource. -func (r _resource) createOrPatchWithReplicas(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { +// are taken from the passed in replicas and not from the etcd component. +func (r _resource) createOrPatchWithReplicas(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) mutatingFn := func() error { if builder, err := newStsBuilder(r.client, ctx.Logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { @@ -137,39 +141,42 @@ func (r _resource) createOrPatchWithReplicas(ctx resource.OperatorContext, etcd return nil } -// createOrPatch updates StatefulSet taking changes from passed in etcd resource. -func (r _resource) createOrPatch(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd) error { +// createOrPatch updates StatefulSet taking changes from passed in etcd component. +func (r _resource) createOrPatch(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { return r.createOrPatchWithReplicas(ctx, etcd, etcd.Spec.Replicas) } -func (r _resource) handleImmutableFieldUpdates(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - if existingSts.Generation > 1 && doesEtcdChangesRequireRecreation(existingSts, etcd) { - ctx.Logger.Info("Immutable fields have been updated, need to recreate StatefulSet", "etcd") - - if err := r.TriggerDelete(ctx, etcd); err != nil { - return fmt.Errorf("error deleting StatefulSet with immutable field updates: %w", err) - } - } - return nil -} - -// doesEtcdChangesRequireRecreation checks if changes have been made to certain fields in the etcd spec -// which will require a recreation of StatefulSet in order to get reflected. Currently, it checks for changes -// in the ServiceName and PodManagementPolicy. -func doesEtcdChangesRequireRecreation(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { - return sts.Spec.ServiceName != etcd.GetPeerServiceName() || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement -} - -func (r _resource) handlePeerTLSEnabled(ctx resource.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - peerTLSEnabled, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) +func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { + peerTLSEnabledForAllMembers, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) if err != nil { return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) } - if isPeerTLSChangedToEnabled(peerTLSEnabled, etcd) { + /** + Case 1: + Scaled from 1 to 3 and peer URL TLS changed enabled or disabled - g/g case + Case 2: + Scaled from 1 to 3 and no change in peer URL TLS (can be kept as HTTP or HTTPS) + Case 3: + Peer URL TLS changed but no change in replicas + */ + + /** + After the changes are made to etcd-wrapper and etcd-backup-restore then we can use the following logic and simplify the handling of peer URL TLS change + if peerTLSNotEnabledOnAllMembers { + if sts.HasAnnotationValue(etcdGenerationAnnotation) != etcd.Spec.Generation { + createOrPathKeepingExistingStsReplicas + } + if !sts.Ready() { + requeue + } + } + */ + + if isPeerTLSChangedToEnabled(peerTLSEnabledForAllMembers, etcd) { ctx.Logger.Info("Attempting to enable TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) - // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd resource due to + // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd component due to // enabling of TLS for peer communication. It preserves the current STS replicas. if err = r.createOrPatchWithReplicas(ctx, etcd, *existingSts.Spec.Replicas); err != nil { return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %w", err) @@ -199,7 +206,7 @@ func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druid return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } -func deleteAllStsPods(ctx resource.OperatorContext, cl client.Client, opName string, sts *appsv1.StatefulSet) error { +func deleteAllStsPods(ctx component.OperatorContext, cl client.Client, opName string, sts *appsv1.StatefulSet) error { // Get all Pods belonging to the StatefulSet podList := &corev1.PodList{} listOpts := []client.ListOption{ diff --git a/internal/operator/statefulset/statefulset_test.go b/internal/operator/statefulset/statefulset_test.go index baf1dc780..7de088238 100644 --- a/internal/operator/statefulset/statefulset_test.go +++ b/internal/operator/statefulset/statefulset_test.go @@ -8,7 +8,7 @@ import ( "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -65,7 +65,7 @@ func TestGetExistingResourceNames(t *testing.T) { } cl := fakeClientBuilder.Build() operator := New(cl, nil, nil) - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) actualStsNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -123,7 +123,7 @@ func TestSyncWhenNoSTSExists(t *testing.T) { features.UseEtcdWrapper: true, }) // *************** Test and assert *************** - opCtx := resource.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) opCtx.Data[common.ConfigMapCheckSumKey] = testutils.TestConfigMapCheckSum syncErr := operator.Sync(opCtx, etcd) latestSTS, getErr := getLatestStatefulSet(cl, etcd) diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go index 69cda6c5a..66c44645d 100644 --- a/internal/utils/concurrent.go +++ b/internal/utils/concurrent.go @@ -5,7 +5,7 @@ import ( "runtime/debug" "sync" - "github.com/gardener/etcd-druid/internal/operator/resource" + "github.com/gardener/etcd-druid/internal/operator/component" ) // OperatorTask is a holder for a named function. @@ -14,13 +14,13 @@ type OperatorTask struct { Name string // Fn is the function which accepts an operator context and returns an error if there is one. // Implementations of Fn should handle context cancellation properly. - Fn func(ctx resource.OperatorContext) error + Fn func(ctx component.OperatorContext) error } // RunConcurrently runs tasks concurrently with number of goroutines bounded by bound. // If there is a panic executing a single OperatorTask then it will capture the panic and capture it as an error // which will then subsequently be returned from this function. It will not propagate the panic causing the app to exit. -func RunConcurrently(ctx resource.OperatorContext, tasks []OperatorTask) []error { +func RunConcurrently(ctx component.OperatorContext, tasks []OperatorTask) []error { rg := newRunGroup(len(tasks)) for _, task := range tasks { rg.trigger(ctx, task) @@ -44,7 +44,7 @@ func newRunGroup(numTasks int) *runGroup { } // trigger executes the task in a go-routine. -func (g *runGroup) trigger(ctx resource.OperatorContext, task OperatorTask) { +func (g *runGroup) trigger(ctx component.OperatorContext, task OperatorTask) { g.wg.Add(1) go func(task OperatorTask) { defer g.wg.Done() diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index 10d35c0fe..f8aacbfaa 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -67,8 +67,9 @@ func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.E return nil, nil } -// FetchPVCWarningEventsForStatefulSet fetches warning events for PVCs for a statefulset and returns them as an error -func FetchPVCWarningEventsForStatefulSet(ctx context.Context, cl client.Client, sts *appsv1.StatefulSet) (string, error) { +// FetchPVCWarningMessageForStatefulSet fetches warning message for PVCs for a statefulset, if found concatenates the first 2 warning messages and returns +// them as string warning message. In case it fails to fetch events, it collects the errors and returns the combined error. +func FetchPVCWarningMessageForStatefulSet(ctx context.Context, cl client.Client, sts *appsv1.StatefulSet) (string, error) { pvcs := &corev1.PersistentVolumeClaimList{} if err := cl.List(ctx, pvcs, client.InNamespace(sts.GetNamespace())); err != nil { return "", fmt.Errorf("unable to list PVCs for sts %s: %v", sts.Name, err) diff --git a/test/integration/controllers/etcd/reconciler_test.go b/test/integration/controllers/etcd/reconciler_test.go index 123110fbc..c21777a0c 100644 --- a/test/integration/controllers/etcd/reconciler_test.go +++ b/test/integration/controllers/etcd/reconciler_test.go @@ -82,8 +82,8 @@ var _ = Describe("Etcd Controller", func() { instance = testutils.EtcdBuilderWithDefaults("foo1", namespace).Build() storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) + errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) + Expect(errs).Should(BeNil()) Expect(k8sClient.Create(context.TODO(), instance)).To(Succeed()) sts = &appsv1.StatefulSet{} @@ -211,8 +211,8 @@ var _ = Describe("Etcd Controller", func() { if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { By("create backup-store secrets") storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) + errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) + Expect(errs).Should(BeNil()) } By("create etcd instance and check if it has been created") @@ -280,8 +280,8 @@ var _ = Describe("Multinode ETCD", func() { BeforeEach(func() { instance = testutils.EtcdBuilderWithDefaults("foo82", namespace).Build() storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) + errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) + Expect(errs).Should(BeNil()) }) It("should create the statefulset based on the replicas in ETCD CR", func() { // First delete existing statefulset if any. @@ -383,8 +383,8 @@ var _ = Describe("Multinode ETCD", func() { if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errors := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) + errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) + Expect(errs).Should(BeNil()) } err = k8sClient.Create(context.TODO(), instance) Expect(err).NotTo(HaveOccurred()) diff --git a/test/it/controller/assets/assets.go b/test/it/controller/assets/assets.go new file mode 100644 index 000000000..dd8de0485 --- /dev/null +++ b/test/it/controller/assets/assets.go @@ -0,0 +1,52 @@ +// Copyright 2023 SAP SE or an SAP affiliate company +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assets + +import ( + "path/filepath" + + "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/gardener/pkg/utils/imagevector" + + . "github.com/onsi/gomega" +) + +// GetEtcdCrdPath returns the path to the Etcd CRD. +func GetEtcdCrdPath() string { + return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "10-crd-druid.gardener.cloud_etcds.yaml") +} + +// GetEtcdCopyBackupsTaskCrdPath returns the path to the EtcdCopyBackupsTask CRD. +func GetEtcdCopyBackupsTaskCrdPath() string { + return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml") +} + +// GetEtcdCopyBackupsBaseChartPath returns the path to the etcd-copy-backups chart. +func GetEtcdCopyBackupsBaseChartPath() string { + return filepath.Join("..", "..", "..", "..", "charts", "etcd-copy-backups") +} + +// GetEtcdChartPath returns the path to the etcd chart. +func GetEtcdChartPath() string { + return filepath.Join("..", "..", "..", "..", "charts", "etcd") +} + +// CreateImageVector creates an image vector. +func CreateImageVector(g *WithT) imagevector.ImageVector { + imageVectorPath := filepath.Join("..", "..", "..", "..", common.ChartPath, "images.yaml") + imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(imageVectorPath) + g.Expect(err).To(BeNil()) + return imageVector +} diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go new file mode 100644 index 000000000..95701b868 --- /dev/null +++ b/test/it/controller/etcd/assertions.go @@ -0,0 +1,257 @@ +package etcd + +import ( + "context" + "fmt" + "slices" + "sync" + "testing" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/operator" + "github.com/gardener/etcd-druid/internal/operator/component" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/etcd-druid/test/it/setup" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type componentCreatedAssertionFn func(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) + +// ReconcilerTestEnv represents the test environment for the etcd reconciler. +type ReconcilerTestEnv struct { + itTestEnv setup.IntegrationTestEnv + itTestEnvCloser setup.IntegrationTestEnvCloser + reconciler *etcd.Reconciler +} + +func getKindToComponentCreatedAssertionFns() map[operator.Kind]componentCreatedAssertionFn { + return map[operator.Kind]componentCreatedAssertionFn{ + operator.MemberLeaseKind: assertMemberLeasesCreated, + operator.SnapshotLeaseKind: assertSnapshotLeasesCreated, + operator.ClientServiceKind: assertClientServiceCreated, + operator.PeerServiceKind: assertPeerServiceCreated, + operator.ConfigMapKind: assertConfigMapCreated, + operator.PodDisruptionBudgetKind: assertPDBCreated, + operator.ServiceAccountKind: assertServiceAccountCreated, + operator.RoleKind: assertRoleCreated, + operator.RoleBindingKind: assertRoleBindingCreated, + operator.StatefulSetKind: assertStatefulSetCreated, + } +} + +// assertAllComponentsCreatedSuccessfully asserts that all components of the etcd resource are created successfully eventually. +func assertAllComponentsCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + kindToAssertionFns := getKindToComponentCreatedAssertionFns() + assertFns := make([]componentCreatedAssertionFn, 0, len(kindToAssertionFns)) + for _, assertFn := range kindToAssertionFns { + assertFns = append(assertFns, assertFn) + } + doAssertComponentsCreatedSuccessfully(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) +} + +func assertSelectedComponentsCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { + g := NewWithT(t) + assertFns := make([]componentCreatedAssertionFn, 0, len(componentKinds)) + for _, kind := range componentKinds { + assertFn, ok := getKindToComponentCreatedAssertionFns()[kind] + g.Expect(ok).To(BeTrue(), fmt.Sprintf("assertion function for %s not found", kind)) + assertFns = append(assertFns, assertFn) + } + doAssertComponentsCreatedSuccessfully(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) +} + +func assertComponentsNotCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { + opRegistry := rtEnv.reconciler.GetOperatorRegistry() + opCtx := component.NewOperatorContext(ctx, rtEnv.itTestEnv.GetLogger(), t.Name()) + for _, kind := range componentKinds { + assertResourceCreation(opCtx, t, opRegistry, kind, etcd, []string{}, timeout, pollInterval) + } +} + +func doAssertComponentsCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, assertionFns []componentCreatedAssertionFn, timeout, pollInterval time.Duration) { + opRegistry := rtEnv.reconciler.GetOperatorRegistry() + opCtx := component.NewOperatorContext(ctx, rtEnv.itTestEnv.GetLogger(), t.Name()) + wg := sync.WaitGroup{} + wg.Add(len(assertionFns)) + for _, assertFn := range assertionFns { + assertFn := assertFn + go func() { + defer wg.Done() + assertFn(opCtx, t, opRegistry, etcd, timeout, pollInterval) + }() + } + wg.Wait() +} + +func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, kind operator.Kind, etcd *druidv1alpha1.Etcd, expectedResourceNames []string, timeout, pollInterval time.Duration) { + g := NewWithT(t) + op := opRegistry.GetOperator(kind) + checkFn := func() error { + actualResourceNames, err := op.GetExistingResourceNames(ctx, etcd) + if err != nil { + return err + } + if len(actualResourceNames) > len(expectedResourceNames) { + return fmt.Errorf("expected only %d %s, found %v", len(expectedResourceNames), kind, actualResourceNames) + } + slices.Sort(actualResourceNames) + slices.Sort(expectedResourceNames) + if !slices.Equal(actualResourceNames, expectedResourceNames) { + return fmt.Errorf("expected %s: %v, found %v instead", kind, expectedResourceNames, actualResourceNames) + } + msg := utils.IfConditionOr[string](len(expectedResourceNames) == 0, + fmt.Sprintf("%s: %v not created successfully", kind, expectedResourceNames), + fmt.Sprintf("%s: %v created successfully", kind, expectedResourceNames)) + t.Log(msg) + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).WithContext(ctx).Should(BeNil()) +} + +func assertMemberLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedMemberLeaseNames := make([]string, 0, etcd.Spec.Replicas) + for i := 0; i < int(etcd.Spec.Replicas); i++ { + expectedMemberLeaseNames = append(expectedMemberLeaseNames, fmt.Sprintf("%s-%d", etcd.Name, i)) + } + assertResourceCreation(ctx, t, opRegistry, operator.MemberLeaseKind, etcd, expectedMemberLeaseNames, timeout, pollInterval) +} + +func assertSnapshotLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedSnapshotLeaseNames := make([]string, 0, 2) + if etcd.IsBackupStoreEnabled() { + expectedSnapshotLeaseNames = []string{etcd.GetDeltaSnapshotLeaseName(), etcd.GetFullSnapshotLeaseName()} + } + assertResourceCreation(ctx, t, opRegistry, operator.SnapshotLeaseKind, etcd, expectedSnapshotLeaseNames, timeout, pollInterval) +} + +func assertClientServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedClientServiceNames := []string{etcd.GetClientServiceName()} + assertResourceCreation(ctx, t, opRegistry, operator.ClientServiceKind, etcd, expectedClientServiceNames, timeout, pollInterval) +} + +func assertPeerServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedPeerServiceNames := []string{etcd.GetPeerServiceName()} + assertResourceCreation(ctx, t, opRegistry, operator.PeerServiceKind, etcd, expectedPeerServiceNames, timeout, pollInterval) +} + +func assertConfigMapCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedConfigMapNames := []string{etcd.GetConfigMapName()} + assertResourceCreation(ctx, t, opRegistry, operator.ConfigMapKind, etcd, expectedConfigMapNames, timeout, pollInterval) +} + +func assertPDBCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedPDBNames := []string{etcd.Name} + assertResourceCreation(ctx, t, opRegistry, operator.PodDisruptionBudgetKind, etcd, expectedPDBNames, timeout, pollInterval) +} + +func assertServiceAccountCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedServiceAccountNames := []string{etcd.GetServiceAccountName()} + assertResourceCreation(ctx, t, opRegistry, operator.ServiceAccountKind, etcd, expectedServiceAccountNames, timeout, pollInterval) +} + +func assertRoleCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedRoleNames := []string{etcd.GetRoleName()} + assertResourceCreation(ctx, t, opRegistry, operator.RoleKind, etcd, expectedRoleNames, timeout, pollInterval) +} + +func assertRoleBindingCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedRoleBindingNames := []string{etcd.GetRoleBindingName()} + assertResourceCreation(ctx, t, opRegistry, operator.RoleBindingKind, etcd, expectedRoleBindingNames, timeout, pollInterval) +} + +func assertStatefulSetCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + expectedSTSNames := []string{etcd.Name} + assertResourceCreation(ctx, t, opRegistry, operator.StatefulSetKind, etcd, expectedSTSNames, timeout, pollInterval) +} + +func assertETCDObservedGeneration(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedObservedGeneration *int64, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + etcdInstance := &druidv1alpha1.Etcd{} + err := cl.Get(context.Background(), etcdObjectKey, etcdInstance) + if err != nil { + return err + } + if expectedObservedGeneration == nil { + if etcdInstance.Status.ObservedGeneration != nil { + return fmt.Errorf("expected observedGeneration to be nil, found %v", etcdInstance.Status.ObservedGeneration) + } + return nil + } else { + if etcdInstance.Status.ObservedGeneration == nil { + return fmt.Errorf("expected observedGeneration to be %v, found nil", *expectedObservedGeneration) + } + if *etcdInstance.Status.ObservedGeneration != *expectedObservedGeneration { + return fmt.Errorf("expected observedGeneration to be %d, found %d", *expectedObservedGeneration, *etcdInstance.Status.ObservedGeneration) + } + } + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + t.Logf("observedGeneration correctly set to %s", logPointerTypeToString[int64](expectedObservedGeneration)) +} + +func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedLastOperation druidv1alpha1.LastOperation, expectedLastErrors []druidv1alpha1.LastError, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + etcdInstance := &druidv1alpha1.Etcd{} + err := cl.Get(context.Background(), etcdObjectKey, etcdInstance) + if err != nil { + return err + } + if etcdInstance.Status.LastOperation.Type != expectedLastOperation.Type && + etcdInstance.Status.LastOperation.State != expectedLastOperation.State { + return fmt.Errorf("expected lastOperation to be %s, found %s", expectedLastOperation, etcdInstance.Status.LastOperation) + } + + // For comparing last errors, it is sufficient to compare their length and their error codes. + expectedErrorCodes := getErrorCodesFromLastErrors(expectedLastErrors) + slices.Sort(expectedErrorCodes) + actualErrorCodes := getErrorCodesFromLastErrors(etcdInstance.Status.LastErrors) + slices.Sort(actualErrorCodes) + + if !slices.Equal(expectedErrorCodes, actualErrorCodes) { + return fmt.Errorf("expected lastErrors to be %v, found %v", expectedLastErrors, etcdInstance.Status.LastErrors) + } + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + t.Log("lastOperation and lastErrors updated successfully") +} + +func assertETCDOperationAnnotationRemovedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + etcdInstance := &druidv1alpha1.Etcd{} + err := cl.Get(context.Background(), etcdObjectKey, etcdInstance) + if err != nil { + return err + } + if metav1.HasAnnotation(etcdInstance.ObjectMeta, v1beta1constants.GardenerOperation) { + return fmt.Errorf("expected reconcile operation annotation to be removed, found %v", v1beta1constants.GardenerOperation) + } + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + t.Log("reconcile operation annotation removed successfully") +} + +func getErrorCodesFromLastErrors(lastErrors []druidv1alpha1.LastError) []druidv1alpha1.ErrorCode { + errorCodes := make([]druidv1alpha1.ErrorCode, 0, len(lastErrors)) + for _, lastErr := range lastErrors { + errorCodes = append(errorCodes, lastErr.Code) + } + return errorCodes +} + +func logPointerTypeToString[T any](val *T) string { + if val == nil { + return "" + } + return fmt.Sprintf("%v", *val) +} diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go new file mode 100644 index 000000000..af37a2b87 --- /dev/null +++ b/test/it/controller/etcd/reconciler_test.go @@ -0,0 +1,181 @@ +package etcd + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "testing" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/features" + "github.com/gardener/etcd-druid/internal/operator" + "github.com/gardener/etcd-druid/test/it/controller/assets" + "github.com/gardener/etcd-druid/test/it/setup" + testutils "github.com/gardener/etcd-druid/test/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + . "github.com/onsi/gomega" + "k8s.io/component-base/featuregate" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const testNamespacePrefix = "etcd-reconciler-test-" + +/* + status update check: + * create etcd resource and let sts be created + * assert the etcd status + * update sts status and then assert etcd status again + + deletion flow + spec reconcile flow + +*/ + +func TestEtcdReconcilerSpecWithoutAutoReconcile(t *testing.T) { + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, false) + defer reconcilerTestEnv.itTestEnvCloser() + + tests := []struct { + name string + fn func(t *testing.T, testNamespace string, reconcilerTestEnv ReconcilerTestEnv) + }{ + //{"should create all managed resources when etcd resource is created", testAllManagedResourcesAreCreated}, + {"should succeed only in creation of some resources and not all and should record error in lastErrors and lastOperation", testFailureToCreateAllResources}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNs := createTestNamespaceName(t) + t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) + reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs) + test.fn(t, testNs, reconcilerTestEnv) + }) + } +} + +func testAllManagedResourcesAreCreated(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { + const ( + timeout = time.Minute * 2 + pollingInterval = time.Second * 2 + ) + // ***************** setup ***************** + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + WithAnnotations(map[string]string{v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile}). + Build() + g.Expect(etcdInstance.Spec.Backup.Store).ToNot(BeNil()) + g.Expect(etcdInstance.Spec.Backup.Store.SecretRef).ToNot(BeNil()) + cl := reconcilerTestEnv.itTestEnv.GetClient() + ctx := context.Background() + // create backup secrets + g.Expect(testutils.CreateSecrets(ctx, cl, testNs, etcdInstance.Spec.Backup.Store.SecretRef.Name)).To(Succeed()) + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + // ***************** test etcd spec reconciliation ***************** + // It is sufficient to test that the resources are created as part of the sync. The configuration of each + // resource is now extensively covered in the unit tests for the respective component operator. + assertAllComponentsCreatedSuccessfully(ctx, t, reconcilerTestEnv, etcdInstance, timeout, pollingInterval) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) + expectedLastOperation := druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: druidv1alpha1.LastOperationStateSucceeded, + } + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, nil, 5*time.Second, 1*time.Second) + assertETCDOperationAnnotationRemovedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 5*time.Second, 1*time.Second) +} + +func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { + const ( + timeout = time.Minute * 3 + pollingInterval = time.Second * 2 + ) + // ***************** setup ***************** + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + WithAnnotations(map[string]string{v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile}). + // The client service label value has an invalid value. ':' are not accepted as valid character in label values. + // This should fail creation of client service and cause a requeue. + WithEtcdClientServiceLabels(map[string]string{"invalid-label": "invalid-label:value"}). + Build() + + ctx := context.Background() + cl := reconcilerTestEnv.itTestEnv.GetClient() + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + // ***************** test etcd spec reconciliation ***************** + componentKindCreated := []operator.Kind{operator.MemberLeaseKind} + assertSelectedComponentsCreatedSuccessfully(ctx, t, reconcilerTestEnv, etcdInstance, componentKindCreated, timeout, pollingInterval) + componentKindNotCreated := []operator.Kind{ + operator.SnapshotLeaseKind, // no backup store has been set + operator.ClientServiceKind, + operator.PeerServiceKind, + operator.ConfigMapKind, + operator.PodDisruptionBudgetKind, + operator.ServiceAccountKind, + operator.RoleKind, + operator.RoleBindingKind, + operator.StatefulSetKind, + } + assertComponentsNotCreatedSuccessfully(ctx, t, reconcilerTestEnv, etcdInstance, componentKindNotCreated, timeout, pollingInterval) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), nil, 5*time.Second, 1*time.Second) + expectedLastOperation := druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: druidv1alpha1.LastOperationStateError, + } + expectedLastErrs := []druidv1alpha1.LastError{ + { + Code: "ERR_SYNC_CLIENT_SERVICE", + }, + } + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, expectedLastErrs, 5*time.Second, 1*time.Second) +} + +func createTestNamespaceName(t *testing.T) string { + b := make([]byte, 4) + _, err := rand.Read(b) + g := NewWithT(t) + g.Expect(err).ToNot(HaveOccurred()) + namespaceSuffix := hex.EncodeToString(b) + return fmt.Sprintf("%s-%s", testNamespacePrefix, namespaceSuffix) +} + +func initializeEtcdReconcilerTestEnv(t *testing.T, autoReconcile bool) ReconcilerTestEnv { + g := NewWithT(t) + itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) + var ( + reconciler *etcd.Reconciler + err error + ) + itTestEnv.RegisterReconciler(func(mgr manager.Manager) { + reconciler, err = etcd.NewReconcilerWithImageVector(mgr, + &etcd.Config{ + Workers: 5, + EnableEtcdSpecAutoReconcile: autoReconcile, + DisableEtcdServiceAccountAutomount: false, + EtcdStatusSyncPeriod: 2 * time.Second, + FeatureGates: map[featuregate.Feature]bool{ + features.UseEtcdWrapper: true, + }, + }, assets.CreateImageVector(g)) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(reconciler.RegisterWithManager(mgr)).To(Succeed()) + }) + itTestEnv.StartManager() + t.Log("successfully registered etcd reconciler with manager and started manager") + return ReconcilerTestEnv{ + itTestEnv: itTestEnv, + itTestEnvCloser: itTestEnvCloser, + reconciler: reconciler, + } +} diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go new file mode 100644 index 000000000..2e146879b --- /dev/null +++ b/test/it/setup/setup.go @@ -0,0 +1,127 @@ +package setup + +import ( + "context" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + eventsv1beta1 "k8s.io/api/events/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +type AddToManagerFn func(mgr manager.Manager) + +type IntegrationTestEnv interface { + RegisterReconciler(addToMgrFn AddToManagerFn) + StartManager() + GetClient() client.Client + GetConfig() *rest.Config + CreateTestNamespace(name string) + GetLogger() logr.Logger +} + +type itTestEnv struct { + ctx context.Context + cancelFn context.CancelFunc + g *WithT + mgr manager.Manager + client client.Client + config *rest.Config + testEnv *envtest.Environment + logger logr.Logger +} + +type IntegrationTestEnvCloser func() + +func NewIntegrationTestEnv(t *testing.T, loggerName string, crdDirectoryPaths []string) (IntegrationTestEnv, IntegrationTestEnvCloser) { + ctx, cancelFunc := context.WithCancel(context.Background()) + itEnv := &itTestEnv{ + ctx: ctx, + cancelFn: cancelFunc, + g: NewWithT(t), + logger: ctrl.Log.WithName(loggerName), + } + druidScheme := itEnv.prepareScheme() + itEnv.createTestEnvironment(druidScheme, crdDirectoryPaths) + itEnv.createManager() + return itEnv, func() { + itEnv.cancelFn() + itEnv.g.Expect(itEnv.testEnv.Stop()).To(Succeed()) + } +} + +func (t *itTestEnv) RegisterReconciler(addToMgrFn AddToManagerFn) { + addToMgrFn(t.mgr) +} + +func (t *itTestEnv) StartManager() { + go func() { + t.g.Expect(t.mgr.Start(t.ctx)).To(Succeed()) + }() +} + +func (t *itTestEnv) GetClient() client.Client { + return t.client +} + +func (t *itTestEnv) GetConfig() *rest.Config { + return t.config +} + +func (t *itTestEnv) CreateTestNamespace(name string) { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + t.g.Expect(t.client.Create(t.ctx, ns)).To(Succeed()) +} + +func (t *itTestEnv) GetLogger() logr.Logger { + return t.logger +} + +func (t *itTestEnv) prepareScheme() *k8sruntime.Scheme { + t.g.Expect(druidv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) + return scheme.Scheme +} + +func (t *itTestEnv) createTestEnvironment(scheme *k8sruntime.Scheme, crdDirectoryPaths []string) { + testEnv := &envtest.Environment{ + Scheme: scheme, + ErrorIfCRDPathMissing: true, + CRDDirectoryPaths: crdDirectoryPaths, + } + + cfg, err := testEnv.Start() + t.g.Expect(err).ToNot(HaveOccurred()) + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + t.g.Expect(err).ToNot(HaveOccurred()) + t.config = cfg + t.testEnv = testEnv + t.client = cl +} + +func (t *itTestEnv) createManager() { + mgr, err := manager.New(t.config, manager.Options{ + MetricsBindAddress: "0", + ClientDisableCacheFor: []client.Object{ + &corev1.Event{}, + &eventsv1beta1.Event{}, + &eventsv1.Event{}, + }, + }) + t.g.Expect(err).ToNot(HaveOccurred()) + t.mgr = mgr +} diff --git a/test/utils/constants.go b/test/utils/constants.go index 7e4ce1dc8..3963b5372 100644 --- a/test/utils/constants.go +++ b/test/utils/constants.go @@ -1,9 +1,9 @@ package utils const ( - TestEtcdName = "test-etcd" + TestEtcdName = "etcd-test" // TestNamespace is a test namespace to be used in tests. - TestNamespace = "test-namespace" + TestNamespace = "test-ns" ) // Image vector constants diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 730644496..2edfbafc2 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -289,6 +289,13 @@ func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { return eb } +// WithAnnotations merges the existing annotations if any with the annotations passed. +// Any existing entry will be replaced by the one that is present in the annotations passed to this method. +func (eb *EtcdBuilder) WithAnnotations(annotations map[string]string) *EtcdBuilder { + eb.etcd.Annotations = MergeMaps[string, string](eb.etcd.Annotations, annotations) + return eb +} + // WithDefaultBackup creates a default backup spec and initializes etcd with it. func (eb *EtcdBuilder) WithDefaultBackup() *EtcdBuilder { eb.etcd.Spec.Backup = getBackupSpec() diff --git a/test/utils/secret.go b/test/utils/secret.go index 127269cbc..87ecb0c69 100644 --- a/test/utils/secret.go +++ b/test/utils/secret.go @@ -6,6 +6,7 @@ package utils import ( "context" + "errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -14,8 +15,8 @@ import ( ) // CreateSecrets creates all the given secrets -func CreateSecrets(ctx context.Context, c client.Client, namespace string, secrets ...string) []error { - var errors []error +func CreateSecrets(ctx context.Context, c client.Client, namespace string, secrets ...string) error { + var createErrs error for _, name := range secrets { secret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -31,8 +32,8 @@ func CreateSecrets(ctx context.Context, c client.Client, namespace string, secre continue } if err != nil { - errors = append(errors, err) + createErrs = errors.Join(createErrs, err) } } - return errors + return createErrs } From c65c5ea4a411e9ec5a689482467e096fbb43ccaf Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 8 Feb 2024 16:50:52 +0530 Subject: [PATCH 085/235] removed fakeclient and introduced test client, adapted unit tests, wrote it tests for delete etcd flow --- internal/controller/etcd/reconcile_delete.go | 2 +- internal/controller/etcd/reconciler.go | 1 + .../clientservice/clientservice_test.go | 37 +-- internal/operator/configmap/configmap_test.go | 30 +-- .../operator/memberlease/memberlease_test.go | 26 +- .../operator/peerservice/peerservice_test.go | 19 +- .../poddisruptionbudget_test.go | 20 +- internal/operator/role/role_test.go | 23 +- .../operator/rolebinding/rolebinding_test.go | 21 +- .../serviceaccount/serviceaccount_test.go | 14 +- .../snapshotlease/snapshotlease_test.go | 35 ++- internal/operator/statefulset/statefulset.go | 21 +- .../operator/statefulset/statefulset_test.go | 13 +- .../operator/statefulset}/stsmatcher.go | 17 +- internal/utils/lease_test.go | 11 +- internal/utils/statefulset_test.go | 7 +- internal/utils/store_test.go | 7 +- test/it/controller/etcd/assertions.go | 104 ++++++-- test/it/controller/etcd/reconciler_test.go | 210 ++++++++++++++-- test/it/setup/setup.go | 39 ++- test/utils/client.go | 235 ++++++++++++++++++ test/utils/fakeclient.go | 135 ---------- 22 files changed, 674 insertions(+), 353 deletions(-) rename {test/utils => internal/operator/statefulset}/stsmatcher.go (96%) create mode 100644 test/utils/client.go delete mode 100644 test/utils/fakeclient.go diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 5cda2510f..b7d4d4900 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -36,7 +36,7 @@ func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjK return result } operators := r.operatorRegistry.AllOperators() - deleteTasks := make([]utils.OperatorTask, len(operators)) + deleteTasks := make([]utils.OperatorTask, 0, len(operators)) for kind, operator := range operators { operator := operator deleteTasks = append(deleteTasks, utils.OperatorTask{ diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index ef9535d55..ab808489e 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -119,6 +119,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult() } +// GetOperatorRegistry returns the operator registry. func (r *Reconciler) GetOperatorRegistry() operator.Registry { return r.operatorRegistry } diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index af1b85a0f..6068bf3bd 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -20,14 +20,13 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" - . "github.com/onsi/gomega/gstruct" - "k8s.io/utils/pointer" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" + . "github.com/onsi/gomega/gstruct" + "k8s.io/utils/pointer" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -49,6 +48,7 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return the existing service name", svcExists: true, + getErr: nil, expectedServiceNames: []string{etcd.GetClientServiceName()}, }, { @@ -74,11 +74,8 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) - if tc.svcExists { - fakeClientBuilder.WithObjects(newClientService(etcd)) - } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, []client.Object{newClientService(etcd)}, client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -124,8 +121,8 @@ func TestSyncWhenNoServiceExists(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() etcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) @@ -177,10 +174,7 @@ func TestSyncWhenServiceExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // ********************* Setup ********************* - cl := testutils.NewFakeClientBuilder(). - WithPatchError(tc.patchErr). - WithObjects(newClientService(existingEtcd)). - Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newClientService(existingEtcd)}, client.ObjectKey{Name: existingEtcd.GetClientServiceName(), Namespace: existingEtcd.Namespace}) // ********************* test sync with updated ports ********************* operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -234,14 +228,13 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // ********************* Setup ********************* - cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() - operator := New(cl) - opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + var existingObjects []client.Object if tc.svcExists { - syncErr := operator.Sync(opCtx, etcd) - g.Expect(syncErr).ToNot(HaveOccurred()) - ensureClientServiceExists(g, cl, etcd) + existingObjects = append(existingObjects, newClientService(etcd)) } + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}) + operator := New(cl) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) // ********************* Test trigger delete ********************* triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestClientService, getErr := getLatestClientService(cl, etcd) @@ -326,12 +319,6 @@ func newClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { return svc } -func ensureClientServiceExists(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) { - svc, err := getLatestClientService(cl, etcd) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(svc).ToNot(BeNil()) -} - func getLatestClientService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { svc := &corev1.Service{} err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}, svc) diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 928c60207..8cee3d8ef 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -59,11 +59,12 @@ func TestGetExistingResourceNames(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.cmExists { - fakeClientBuilder.WithObjects(newConfigMap(g, etcd)) + existingObjects = append(existingObjects, newConfigMap(g, etcd)) } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) cmNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -116,7 +117,7 @@ func TestSyncWhenNoConfigMapExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := buildEtcd(tc.etcdReplicas, tc.clientTLSEnabled, tc.peerTLSEnabled) - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) @@ -210,10 +211,7 @@ func TestSyncWhenConfigMapExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { originalEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().Build() - cl := testutils.NewFakeClientBuilder(). - WithPatchError(tc.patchErr). - WithObjects(newConfigMap(g, originalEtcd)). - Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newConfigMap(g, originalEtcd)}, getObjectKey(originalEtcd)) updatedEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().WithPeerTLS().Build() updatedEtcd.UID = originalEtcd.UID operator := New(cl) @@ -267,14 +265,13 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // ********************* Setup ********************* - cl := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr).Build() - operator := New(cl) - opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) + var existingObjects []client.Object if tc.cmExists { - syncErr := operator.Sync(opCtx, etcd) - g.Expect(syncErr).ToNot(HaveOccurred()) - ensureConfigMapExists(g, cl, etcd) + existingObjects = append(existingObjects, newConfigMap(g, etcd)) } + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) + operator := New(cl) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) // ********************* Test trigger delete ********************* triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) latestConfigMap, getErr := getLatestConfigMap(cl, etcd) @@ -350,7 +347,8 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), @@ -377,13 +375,11 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), "auto-tls": Equal(false), }), - "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort))), "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))))), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "advertise-client-urls": Equal(fmt.Sprintf("http@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort))), "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))))), })) diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index d669cc7ff..eadbd8b76 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -18,7 +18,6 @@ import ( . "github.com/onsi/gomega/gstruct" gomegatypes "github.com/onsi/gomega/types" coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -36,7 +35,6 @@ func TestGetExistingResourceNames(t *testing.T) { name string etcdReplicas int32 numExistingLeases int - getErr *apierrors.StatusError listErr *apierrors.StatusError expectedErr *druiderr.DruidError }{ @@ -57,7 +55,6 @@ func TestGetExistingResourceNames(t *testing.T) { }, { name: "should return an empty slice when no member leases are found", - getErr: apierrors.NewNotFound(corev1.Resource("leases"), ""), etcdReplicas: 3, numExistingLeases: 0, }, @@ -78,17 +75,16 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder(). - WithGetError(tc.getErr). - WithListError(tc.listErr) + var existingObjects []client.Object if tc.numExistingLeases > 0 { leases, err := newMemberLeases(etcd, tc.numExistingLeases) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { - fakeClientBuilder.WithObjects(lease) + existingObjects = append(existingObjects, lease) } } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd), existingObjects...) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) memberLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -155,15 +151,15 @@ func TestSync(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // *************** set up existing environment ***************** etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).WithGetError(tc.getErr) + var existingObjects []client.Object if tc.numExistingLeases > 0 { leases, err := newMemberLeases(etcd, tc.numExistingLeases) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { - fakeClientBuilder.WithObjects(lease) + existingObjects = append(existingObjects, lease) } } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, tc.createErr, nil, nil, existingObjects, getObjectKeys(etcd)...) // ***************** Setup updated etcd to be passed to the Sync method ***************** var updatedEtcd *druidv1alpha1.Etcd @@ -238,18 +234,18 @@ func TestTriggerDelete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // *************** set up existing environment ***************** etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(tc.etcdReplicas).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteAllOfError(tc.deleteAllOfErr) + var existingObjects []client.Object if tc.numExistingLeases > 0 { leases, err := newMemberLeases(etcd, tc.numExistingLeases) g.Expect(err).ToNot(HaveOccurred()) for _, lease := range leases { - fakeClientBuilder.WithObjects(lease) + existingObjects = append(existingObjects, lease) } } for _, nonTargetLeaseName := range nonTargetLeaseNames { - fakeClientBuilder.WithObjects(testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.MemberLeaseComponentName)) + existingObjects = append(existingObjects, testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.MemberLeaseComponentName)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd), existingObjects...) // ***************** Setup operator and test ***************** operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 7c5aefda7..3fce059df 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -73,11 +73,12 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.svcExists { - fakeClientBuilder.WithObjects(newPeerService(etcd)) + existingObjects = append(existingObjects, newPeerService(etcd)) } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -120,11 +121,11 @@ func TestSyncWhenNoServiceExists(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() if tc.createWithPort != nil { etcdBuilder.WithEtcdServerPort(tc.createWithPort) } etcd := etcdBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -170,9 +171,7 @@ func TestSyncWhenServiceExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { existingEtcd := etcdBuilder.Build() - cl := testutils.NewFakeClientBuilder().WithPatchError(tc.patchErr). - WithObjects(newPeerService(existingEtcd)). - Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newPeerService(existingEtcd)}, getObjectKey(existingEtcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithEtcdServerPort(tc.updateWithPort).Build() @@ -224,11 +223,7 @@ func TestPeerServiceTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) - if tc.svcExists { - fakeClientBuilder.WithObjects(newPeerService(etcd)) - } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, []client.Object{newPeerService(etcd)}, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 8a54b9880..1823082a8 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -58,11 +58,12 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.pdbExists { - fakeClientBuilder.WithObjects(newPodDisruptionBudget(etcd)) + existingObjects = append(existingObjects, newPodDisruptionBudget(etcd)) } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) pdbNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -111,7 +112,7 @@ func TestSyncWhenNoPDBExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -164,10 +165,7 @@ func TestSyncWhenPDBExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { existingEtcd := etcdBuilder.WithReplicas(tc.originalEtcdReplicas).Build() - cl := testutils.NewFakeClientBuilder(). - WithPatchError(tc.patchErr). - WithObjects(newPodDisruptionBudget(existingEtcd)). - Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newPodDisruptionBudget(existingEtcd)}, getObjectKey(existingEtcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithReplicas(tc.updatedEtcdReplicas).Build() @@ -217,11 +215,11 @@ func TestTriggerDelete(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) + var existingObjects []client.Object if tc.pdbExists { - fakeClientBuilder.WithObjects(newPodDisruptionBudget(etcd)) + existingObjects = append(existingObjects, newPodDisruptionBudget(etcd)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 2898f0606..39fc6d02c 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -56,14 +56,12 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } + var existingObjects []client.Object if tc.roleExists { - fakeClientBuilder.WithObjects(newRole(etcd)) + existingObjects = append(existingObjects, newRole(etcd)) } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) roleNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -103,9 +101,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder(). - WithCreateError(tc.createErr). - Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -157,14 +153,11 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.deleteErr != nil { - fakeClientBuilder.WithDeleteError(tc.deleteErr) - } + var existingObjects []client.Object if tc.roleExists { - fakeClientBuilder.WithObjects(newRole(etcd)) + existingObjects = append(existingObjects, newRole(etcd)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) deleteErr := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index 4637aa675..bf29092ec 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -57,14 +57,12 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.getErr != nil { - fakeClientBuilder.WithGetError(tc.getErr) - } + var existingObjects []client.Object if tc.roleBindingExists { - fakeClientBuilder.WithObjects(newRoleBinding(etcd)) + existingObjects = append(existingObjects, newRoleBinding(etcd)) } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) roleBindingNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -104,7 +102,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -156,14 +154,11 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder() - if tc.deleteErr != nil { - fakeClientBuilder.WithDeleteError(tc.deleteErr) - } + var existingObjects []client.Object if tc.roleBindingExists { - fakeClientBuilder.WithObjects(newRoleBinding(etcd)) + existingObjects = append(existingObjects, newRoleBinding(etcd)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index f6a34fdec..d22e67d5c 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -55,11 +55,11 @@ func TestGetExistingResourceNames(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.saExists { - fakeClientBuilder.WithObjects(newServiceAccount(etcd, false)) + existingObjects = append(existingObjects, newServiceAccount(etcd, false)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) operator := New(cl, true) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) saNames, err := operator.GetExistingResourceNames(opCtx, etcd) @@ -109,8 +109,8 @@ func TestSync(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) operator := New(cl, tc.disableAutoMount) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -160,11 +160,11 @@ func TestTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - fakeClientBuilder := testutils.NewFakeClientBuilder().WithDeleteError(tc.deleteErr) + var existingObjects []client.Object if tc.saExists { - fakeClientBuilder.WithObjects(newServiceAccount(etcd, false)) + existingObjects = append(existingObjects, newServiceAccount(etcd, false)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) operator := New(cl, false) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index b46c70f32..4373cd490 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -69,11 +69,12 @@ func TestGetExistingResourceNames(t *testing.T) { etcdBuilder.WithDefaultBackup() } etcd := etcdBuilder.Build() - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.backupEnabled { - fakeClientBuilder.WithObjects(newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) + existingObjects = append(existingObjects, newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) } - operator := New(fakeClientBuilder.Build()) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKeys(etcd)...) + operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) actualSnapshotLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) if tc.expectedErr != nil { @@ -112,7 +113,7 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder().WithCreateError(tc.createErr).Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKeys(etcd)...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -156,14 +157,13 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.NewFakeClientBuilder(). - WithDeleteAllOfError(tc.deleteAllOfErr). - WithObjects( - newDeltaSnapshotLease(existingEtcd), - newFullSnapshotLease(existingEtcd), - newDeltaSnapshotLease(nonTargetEtcd), - newFullSnapshotLease(nonTargetEtcd)). - Build() + existingObjects := []client.Object{ + newDeltaSnapshotLease(existingEtcd), + newFullSnapshotLease(existingEtcd), + newDeltaSnapshotLease(nonTargetEtcd), + newFullSnapshotLease(nonTargetEtcd), + } + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, existingEtcd.Namespace, getSelectorLabelsForAllSnapshotLeases(existingEtcd), existingObjects...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, updatedEtcd) @@ -221,16 +221,11 @@ func TestTriggerDelete(t *testing.T) { etcdBuilder.WithDefaultBackup() } etcd := etcdBuilder.Build() - fakeClientBuilder := testutils.NewFakeClientBuilder(). - WithDeleteAllOfError(tc.deleteAllErr). - WithObjects( - newDeltaSnapshotLease(nonTargetEtcd), - newFullSnapshotLease(nonTargetEtcd), - ) + existingObjects := []client.Object{newDeltaSnapshotLease(nonTargetEtcd), newFullSnapshotLease(nonTargetEtcd)} if tc.backupEnabled { - fakeClientBuilder.WithObjects(newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) + existingObjects = append(existingObjects, newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllErr, nil, etcd.Namespace, getSelectorLabelsForAllSnapshotLeases(etcd), existingObjects...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index bb3040d45..1f4358bcf 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -42,7 +42,7 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) - sts, err := r.getExistingStatefulSet(ctx, objectKey) + sts, err := r.getExistingStatefulSet(ctx, etcd) if err != nil { return nil, druiderr.WrapError(err, ErrGetStatefulSet, @@ -64,7 +64,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) err error ) objectKey := getObjectKey(etcd) - if existingSTS, err = r.getExistingStatefulSet(ctx, objectKey); err != nil { + if existingSTS, err = r.getExistingStatefulSet(ctx, etcd); err != nil { return druiderr.WrapError(err, ErrSyncStatefulSet, "Sync", @@ -89,7 +89,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of StatefulSet", "objectKey", objectKey) - err := r.client.Delete(ctx, emptyStatefulSet(objectKey)) + err := r.client.Delete(ctx, emptyStatefulSet(etcd)) if err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No StatefulSet found, Deletion is a No-Op", "objectKey", objectKey.Name) @@ -104,9 +104,9 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, objectKey client.ObjectKey) (*appsv1.StatefulSet, error) { - sts := emptyStatefulSet(objectKey) - if err := r.client.Get(ctx, objectKey, sts); err != nil { +func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { + sts := emptyStatefulSet(etcd) + if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { if errors.IsNotFound(err) { return nil, nil } @@ -118,7 +118,7 @@ func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, objectK // createOrPatchWithReplicas ensures that the StatefulSet is updated with all changes from passed in etcd but the replicas set on the StatefulSet // are taken from the passed in replicas and not from the etcd component. func (r _resource) createOrPatchWithReplicas(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { - desiredStatefulSet := emptyStatefulSet(getObjectKey(etcd)) + desiredStatefulSet := emptyStatefulSet(etcd) mutatingFn := func() error { if builder, err := newStsBuilder(r.client, ctx.Logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { return druiderr.WrapError(err, @@ -229,11 +229,12 @@ func deleteAllStsPods(ctx component.OperatorContext, cl client.Client, opName st return nil } -func emptyStatefulSet(objectKey client.ObjectKey) *appsv1.StatefulSet { +func emptyStatefulSet(etcd *druidv1alpha1.Etcd) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: objectKey.Name, - Namespace: objectKey.Namespace, + Name: etcd.Name, + Namespace: etcd.Namespace, + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, }, } } diff --git a/internal/operator/statefulset/statefulset_test.go b/internal/operator/statefulset/statefulset_test.go index 7de088238..d5eebea56 100644 --- a/internal/operator/statefulset/statefulset_test.go +++ b/internal/operator/statefulset/statefulset_test.go @@ -59,11 +59,11 @@ func TestGetExistingResourceNames(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.stsExists { - fakeClientBuilder.WithObjects(emptyStatefulSet(getObjectKey(etcd))) + existingObjects = append(existingObjects, emptyStatefulSet(etcd)) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) operator := New(cl, nil, nil) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) actualStsNames, err := operator.GetExistingResourceNames(opCtx, etcd) @@ -111,14 +111,11 @@ func TestSyncWhenNoSTSExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // *************** Build test environment *************** - cl := testutils.NewFakeClientBuilder(). - WithCreateError(tc.createErr). - WithObjects(buildBackupSecret()). - Build() etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(tc.replicas).Build() + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, []client.Object{buildBackupSecret()}, getObjectKey(etcd)) etcdImage, etcdBRImage, initContainerImage, err := utils.GetEtcdImages(etcd, iv, true) g.Expect(err).ToNot(HaveOccurred()) - stsMatcher := testutils.NewStatefulSetMatcher(g, cl, etcd, tc.replicas, true, initContainerImage, etcdImage, etcdBRImage, pointer.String(utils.Local)) + stsMatcher := NewStatefulSetMatcher(g, cl, etcd, tc.replicas, true, initContainerImage, etcdImage, etcdBRImage, pointer.String(utils.Local)) operator := New(cl, iv, map[featuregate.Feature]bool{ features.UseEtcdWrapper: true, }) diff --git a/test/utils/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go similarity index 96% rename from test/utils/stsmatcher.go rename to internal/operator/statefulset/stsmatcher.go index 1f34e736e..366a8271a 100644 --- a/test/utils/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -1,4 +1,4 @@ -package utils +package statefulset import ( "context" @@ -8,6 +8,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/utils" + utils2 "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -81,15 +82,15 @@ func (s StatefulSetMatcher) matchSTSObjectMeta() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "Name": Equal(s.etcd.Name), "Namespace": Equal(s.etcd.Namespace), - "OwnerReferences": MatchEtcdOwnerReference(s.etcd.Name, s.etcd.UID), - "Labels": MatchResourceLabels(getStatefulSetLabels(s.etcd.Name)), + "OwnerReferences": utils2.MatchEtcdOwnerReference(s.etcd.Name, s.etcd.UID), + "Labels": utils2.MatchResourceLabels(getStatefulSetLabels(s.etcd.Name)), }) } func (s StatefulSetMatcher) matchSpec() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "Replicas": PointTo(Equal(s.replicas)), - "Selector": MatchSpecLabelSelector(s.etcd.GetDefaultLabels()), + "Selector": utils2.MatchSpecLabelSelector(s.etcd.GetDefaultLabels()), "PodManagementPolicy": Equal(appsv1.ParallelPodManagement), "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), @@ -127,9 +128,9 @@ func (s StatefulSetMatcher) matchPodTemplateSpec() gomegatypes.GomegaMatcher { func (s StatefulSetMatcher) matchPodObjectMeta() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ - "Labels": MatchResourceLabels(utils.MergeMaps[string, string](getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), - "Annotations": MatchResourceAnnotations(utils.MergeMaps[string, string](s.etcd.Spec.Annotations, map[string]string{ - "checksum/etcd-configmap": TestConfigMapCheckSum, + "Labels": utils2.MatchResourceLabels(utils.MergeMaps[string, string](getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), + "Annotations": utils2.MatchResourceAnnotations(utils.MergeMaps[string, string](s.etcd.Spec.Annotations, map[string]string{ + "checksum/etcd-configmap": utils2.TestConfigMapCheckSum, })), }) } @@ -238,7 +239,7 @@ func (s StatefulSetMatcher) matchEtcdContainerVolMounts() gomegatypes.GomegaMatc } func (s StatefulSetMatcher) matchBackupRestoreContainer() gomegatypes.GomegaMatcher { - containerResources := TypeDeref(s.etcd.Spec.Backup.Resources, defaultTestContainerResources) + containerResources := utils2.TypeDeref(s.etcd.Spec.Backup.Resources, defaultTestContainerResources) return MatchFields(IgnoreExtras, Fields{ "Name": Equal("backup-restore"), "Image": Equal(s.etcdBRImage), diff --git a/internal/utils/lease_test.go b/internal/utils/lease_test.go index 57db730fe..bdaf0cfdc 100644 --- a/internal/utils/lease_test.go +++ b/internal/utils/lease_test.go @@ -16,6 +16,7 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) func TestIsPeerURLTLSEnabledForAllMembers(t *testing.T) { @@ -57,11 +58,15 @@ func TestIsPeerURLTLSEnabledForAllMembers(t *testing.T) { logger := logr.Discard() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithListError(tc.listErr) + var existingObjects []client.Object for _, l := range createLeases(testutils.TestNamespace, testutils.TestEtcdName, etcdReplicas, tc.numETCDMembersWithTLSEnabled) { - fakeClientBuilder.WithObjects(l) + existingObjects = append(existingObjects, l) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, testutils.TestNamespace, map[string]string{ + druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelPartOfKey: testutils.TestEtcdName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + }, existingObjects...) tlsEnabled, err := IsPeerURLTLSEnabledForAllMembers(context.Background(), cl, logger, testutils.TestNamespace, testutils.TestEtcdName) if tc.expectedErr != nil { g.Expect(err).To(Equal(tc.expectedErr)) diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index 48e516a39..e317982ad 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/uuid" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -146,16 +147,16 @@ func TestGetStatefulSet(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithListError(tc.listErr) + var existingObjects []client.Object if tc.isStsPresent { etcdUID := etcd.UID if !tc.ownedByEtcd { etcdUID = uuid.NewUUID() } sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcdUID, etcd.Spec.Replicas) - fakeClientBuilder.WithObjects(sts) + existingObjects = append(existingObjects, sts) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, etcd.GetDefaultLabels(), existingObjects...) foundSts, err := GetStatefulSet(context.Background(), cl, etcd) if tc.expectedErr != nil { g.Expect(err).To(HaveOccurred()) diff --git a/internal/utils/store_test.go b/internal/utils/store_test.go index 27527b981..c502f9893 100644 --- a/internal/utils/store_test.go +++ b/internal/utils/store_test.go @@ -13,6 +13,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) func TestGetHostMountPathFromSecretRef(t *testing.T) { @@ -74,12 +75,12 @@ func TestGetHostMountPathFromSecretRef(t *testing.T) { t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fakeClientBuilder := testutils.NewFakeClientBuilder().WithGetError(tc.getErr) + var existingObjects []client.Object if tc.secretExists { sec := createSecret(existingSecretName, testutils.TestNamespace, tc.hostPathInSecret) - fakeClientBuilder.WithObjects(sec) + existingObjects = append(existingObjects, sec) } - cl := fakeClientBuilder.Build() + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, client.ObjectKey{Name: existingSecretName, Namespace: testutils.TestNamespace}) secretName := IfConditionOr[string](tc.secretExists, existingSecretName, nonExistingSecretName) storeSpec := createStoreSpec(tc.secretRefDefined, secretName, testutils.TestNamespace) actualHostPath, err := GetHostMountPathFromSecretRef(context.Background(), cl, logger, storeSpec, testutils.TestNamespace) diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index 95701b868..a7c27c73f 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -9,6 +9,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/component" @@ -16,17 +17,33 @@ import ( "github.com/gardener/etcd-druid/test/it/setup" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) +var ( + allComponentKinds = []operator.Kind{ + operator.MemberLeaseKind, + operator.SnapshotLeaseKind, + operator.ClientServiceKind, + operator.PeerServiceKind, + operator.ConfigMapKind, + operator.PodDisruptionBudgetKind, + operator.ServiceAccountKind, + operator.RoleKind, + operator.RoleBindingKind, + operator.StatefulSetKind, + } +) + type componentCreatedAssertionFn func(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) // ReconcilerTestEnv represents the test environment for the etcd reconciler. type ReconcilerTestEnv struct { - itTestEnv setup.IntegrationTestEnv - itTestEnvCloser setup.IntegrationTestEnvCloser - reconciler *etcd.Reconciler + itTestEnv setup.IntegrationTestEnv + reconciler *etcd.Reconciler } func getKindToComponentCreatedAssertionFns() map[operator.Kind]componentCreatedAssertionFn { @@ -44,17 +61,21 @@ func getKindToComponentCreatedAssertionFns() map[operator.Kind]componentCreatedA } } -// assertAllComponentsCreatedSuccessfully asserts that all components of the etcd resource are created successfully eventually. -func assertAllComponentsCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertNoComponentsExist(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { + assertComponentsDoNotExist(ctx, t, rtEnv, etcd, allComponentKinds, timeout, pollInterval) +} + +// assertAllComponentsExists asserts that all components of the etcd resource are created successfully eventually. +func assertAllComponentsExists(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { kindToAssertionFns := getKindToComponentCreatedAssertionFns() assertFns := make([]componentCreatedAssertionFn, 0, len(kindToAssertionFns)) for _, assertFn := range kindToAssertionFns { assertFns = append(assertFns, assertFn) } - doAssertComponentsCreatedSuccessfully(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) + doAssertComponentsExist(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) } -func assertSelectedComponentsCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { +func assertSelectedComponentsExists(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { g := NewWithT(t) assertFns := make([]componentCreatedAssertionFn, 0, len(componentKinds)) for _, kind := range componentKinds { @@ -62,10 +83,10 @@ func assertSelectedComponentsCreatedSuccessfully(ctx context.Context, t *testing g.Expect(ok).To(BeTrue(), fmt.Sprintf("assertion function for %s not found", kind)) assertFns = append(assertFns, assertFn) } - doAssertComponentsCreatedSuccessfully(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) + doAssertComponentsExist(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) } -func assertComponentsNotCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { +func assertComponentsDoNotExist(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { opRegistry := rtEnv.reconciler.GetOperatorRegistry() opCtx := component.NewOperatorContext(ctx, rtEnv.itTestEnv.GetLogger(), t.Name()) for _, kind := range componentKinds { @@ -73,7 +94,7 @@ func assertComponentsNotCreatedSuccessfully(ctx context.Context, t *testing.T, r } } -func doAssertComponentsCreatedSuccessfully(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, assertionFns []componentCreatedAssertionFn, timeout, pollInterval time.Duration) { +func doAssertComponentsExist(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, assertionFns []componentCreatedAssertionFn, timeout, pollInterval time.Duration) { opRegistry := rtEnv.reconciler.GetOperatorRegistry() opCtx := component.NewOperatorContext(ctx, rtEnv.itTestEnv.GetLogger(), t.Name()) wg := sync.WaitGroup{} @@ -105,8 +126,8 @@ func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegis return fmt.Errorf("expected %s: %v, found %v instead", kind, expectedResourceNames, actualResourceNames) } msg := utils.IfConditionOr[string](len(expectedResourceNames) == 0, - fmt.Sprintf("%s: %v not created successfully", kind, expectedResourceNames), - fmt.Sprintf("%s: %v created successfully", kind, expectedResourceNames)) + fmt.Sprintf("%s: %v does not exist", kind, expectedResourceNames), + fmt.Sprintf("%s: %v exists", kind, expectedResourceNames)) t.Log(msg) return nil } @@ -196,7 +217,7 @@ func assertETCDObservedGeneration(t *testing.T, cl client.Client, etcdObjectKey t.Logf("observedGeneration correctly set to %s", logPointerTypeToString[int64](expectedObservedGeneration)) } -func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedLastOperation druidv1alpha1.LastOperation, expectedLastErrors []druidv1alpha1.LastError, timeout, pollInterval time.Duration) { +func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedLastOperation *druidv1alpha1.LastOperation, expectedLastErrors []druidv1alpha1.LastError, timeout, pollInterval time.Duration) { g := NewWithT(t) checkFn := func() error { etcdInstance := &druidv1alpha1.Etcd{} @@ -204,7 +225,14 @@ func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl cl if err != nil { return err } - if etcdInstance.Status.LastOperation.Type != expectedLastOperation.Type && + actualLastOperation := etcdInstance.Status.LastOperation + if actualLastOperation == nil { + if expectedLastOperation != nil { + return fmt.Errorf("expected lastOperation to be %s, found nil", expectedLastOperation) + } + return nil + } + if actualLastOperation.Type != expectedLastOperation.Type && etcdInstance.Status.LastOperation.State != expectedLastOperation.State { return fmt.Errorf("expected lastOperation to be %s, found %s", expectedLastOperation, etcdInstance.Status.LastOperation) } @@ -224,7 +252,7 @@ func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl cl t.Log("lastOperation and lastErrors updated successfully") } -func assertETCDOperationAnnotationRemovedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, timeout, pollInterval time.Duration) { +func assertETCDOperationAnnotation(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedAnnotationToBePresent bool, timeout, pollInterval time.Duration) { g := NewWithT(t) checkFn := func() error { etcdInstance := &druidv1alpha1.Etcd{} @@ -232,13 +260,55 @@ func assertETCDOperationAnnotationRemovedSuccessfully(t *testing.T, cl client.Cl if err != nil { return err } - if metav1.HasAnnotation(etcdInstance.ObjectMeta, v1beta1constants.GardenerOperation) { + if metav1.HasAnnotation(etcdInstance.ObjectMeta, v1beta1constants.GardenerOperation) != expectedAnnotationToBePresent { return fmt.Errorf("expected reconcile operation annotation to be removed, found %v", v1beta1constants.GardenerOperation) } return nil } g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) - t.Log("reconcile operation annotation removed successfully") + msg := utils.IfConditionOr[string](expectedAnnotationToBePresent, "reconcile operation annotation present", "reconcile operation annotation removed") + t.Log(msg) +} + +func assertReconcileSuspensionEventRecorded(ctx context.Context, t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + events := &corev1.EventList{} + if err := cl.List(ctx, events, client.InNamespace(etcdObjectKey.Namespace), client.MatchingFields{"involvedObject.name": etcdObjectKey.Name, "type": "Warning"}); err != nil { + return err + } + if len(events.Items) != 1 { + return fmt.Errorf("expected 1 event, found %d", len(events.Items)) + } + if events.Items[0].Reason != "SpecReconciliationSkipped" { + return fmt.Errorf("expected event reason to be SpecReconciliationSkipped, found %s", events.Items[0].Reason) + } + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) +} + +func assertETCDFinalizer(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedFinalizerPresent bool, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + etcdInstance := &druidv1alpha1.Etcd{} + if err := cl.Get(context.Background(), etcdObjectKey, etcdInstance); err != nil { + if apierrors.IsNotFound(err) { + // Once the finalizer is removed, the etcd resource will be removed by k8s very quickly. + // If we find that the resource was indeed removed then this check will pass. + return nil + } + return err + } + finalizerPresent := slices.Contains(etcdInstance.ObjectMeta.Finalizers, common.FinalizerName) + if expectedFinalizerPresent != finalizerPresent { + return fmt.Errorf("expected finalizer to be %s, found %v", utils.IfConditionOr[string](expectedFinalizerPresent, "present", "removed"), etcdInstance.ObjectMeta.Finalizers) + } + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + msg := utils.IfConditionOr[string](expectedFinalizerPresent, "finalizer present", "finalizer removed") + t.Log(msg) } func getErrorCodesFromLastErrors(lastErrors []druidv1alpha1.LastError) []druidv1alpha1.ErrorCode { diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index af37a2b87..e6c2c0a08 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -9,6 +9,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/operator" @@ -16,6 +17,7 @@ import ( "github.com/gardener/etcd-druid/test/it/setup" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" . "github.com/onsi/gomega" "k8s.io/component-base/featuregate" "k8s.io/utils/pointer" @@ -31,23 +33,27 @@ const testNamespacePrefix = "etcd-reconciler-test-" * assert the etcd status * update sts status and then assert etcd status again - deletion flow - spec reconcile flow - + deletion flow -done + spec reconcile flow -done */ +// ------------------------------ reconcile spec tests ------------------------------ func TestEtcdReconcilerSpecWithoutAutoReconcile(t *testing.T) { - reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, false) - defer reconcilerTestEnv.itTestEnvCloser() + itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) + defer itTestEnvCloser(t) + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) tests := []struct { name string fn func(t *testing.T, testNamespace string, reconcilerTestEnv ReconcilerTestEnv) }{ - //{"should create all managed resources when etcd resource is created", testAllManagedResourcesAreCreated}, + {"should create all managed resources when etcd resource is created", testAllManagedResourcesAreCreated}, {"should succeed only in creation of some resources and not all and should record error in lastErrors and lastOperation", testFailureToCreateAllResources}, + {"should not reconcile spec when reconciliation is suspended", testWhenReconciliationIsSuspended}, + {"should not reconcile when no reconcile operation annotation is set", testWhenNoReconcileOperationAnnotationIsSet}, } + t.Parallel() for _, test := range tests { t.Run(test.name, func(t *testing.T) { testNs := createTestNamespaceName(t) @@ -82,14 +88,14 @@ func testAllManagedResourcesAreCreated(t *testing.T, testNs string, reconcilerTe // ***************** test etcd spec reconciliation ***************** // It is sufficient to test that the resources are created as part of the sync. The configuration of each // resource is now extensively covered in the unit tests for the respective component operator. - assertAllComponentsCreatedSuccessfully(ctx, t, reconcilerTestEnv, etcdInstance, timeout, pollingInterval) + assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, timeout, pollingInterval) assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) - expectedLastOperation := druidv1alpha1.LastOperation{ + expectedLastOperation := &druidv1alpha1.LastOperation{ Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateSucceeded, } assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, nil, 5*time.Second, 1*time.Second) - assertETCDOperationAnnotationRemovedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 5*time.Second, 1*time.Second) + assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), false, 5*time.Second, 1*time.Second) } func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { @@ -115,7 +121,7 @@ func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTest g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) // ***************** test etcd spec reconciliation ***************** componentKindCreated := []operator.Kind{operator.MemberLeaseKind} - assertSelectedComponentsCreatedSuccessfully(ctx, t, reconcilerTestEnv, etcdInstance, componentKindCreated, timeout, pollingInterval) + assertSelectedComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, componentKindCreated, timeout, pollingInterval) componentKindNotCreated := []operator.Kind{ operator.SnapshotLeaseKind, // no backup store has been set operator.ClientServiceKind, @@ -127,9 +133,9 @@ func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTest operator.RoleBindingKind, operator.StatefulSetKind, } - assertComponentsNotCreatedSuccessfully(ctx, t, reconcilerTestEnv, etcdInstance, componentKindNotCreated, timeout, pollingInterval) + assertComponentsDoNotExist(ctx, t, reconcilerTestEnv, etcdInstance, componentKindNotCreated, timeout, pollingInterval) assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), nil, 5*time.Second, 1*time.Second) - expectedLastOperation := druidv1alpha1.LastOperation{ + expectedLastOperation := &druidv1alpha1.LastOperation{ Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateError, } @@ -141,6 +147,158 @@ func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTest assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, expectedLastErrs, 5*time.Second, 1*time.Second) } +func testWhenReconciliationIsSuspended(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { + // ***************** setup ***************** + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + WithAnnotations(map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true", + }).Build() + ctx := context.Background() + cl := reconcilerTestEnv.itTestEnv.GetClient() + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + // ***************** test etcd spec reconciliation ***************** + assertNoComponentsExist(ctx, t, reconcilerTestEnv, etcdInstance, 10*time.Second, 2*time.Second) + assertReconcileSuspensionEventRecorded(ctx, t, cl, client.ObjectKeyFromObject(etcdInstance), 10*time.Second, 2*time.Second) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), nil, 5*time.Second, 1*time.Second) + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), nil, nil, 5*time.Second, 1*time.Second) + assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), true, 5*time.Second, 1*time.Second) +} + +func testWhenNoReconcileOperationAnnotationIsSet(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { + // ***************** setup ***************** + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + Build() + ctx := context.Background() + cl := reconcilerTestEnv.itTestEnv.GetClient() + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + // ***************** test etcd spec reconciliation ***************** + assertNoComponentsExist(ctx, t, reconcilerTestEnv, etcdInstance, 10*time.Second, 2*time.Second) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), nil, 5*time.Second, 1*time.Second) + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), nil, nil, 5*time.Second, 1*time.Second) +} + +// ------------------------------ reconcile deletion tests ------------------------------ +func TestDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T) { + // ***************** setup ***************** + itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) + defer itTestEnvCloser(t) + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) + + // --------------------------- create test namespace --------------------------- + testNs := createTestNamespaceName(t) + itTestEnv.CreateTestNamespace(testNs) + t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) + + // ---------------------------- create etcd instance -------------------------- + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + WithAnnotations(map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }).Build() + ctx := context.Background() + cl := reconcilerTestEnv.itTestEnv.GetClient() + createEtcdInstanceWithFinalizer(ctx, t, reconcilerTestEnv, etcdInstance) + + // ***************** test etcd deletion flow ***************** + // mark etcd for deletion + g.Expect(cl.Delete(ctx, etcdInstance)).To(Succeed()) + t.Logf("successfully marked etcd instance for deletion: %s, waiting for resources to be removed...", etcdInstance.Name) + assertNoComponentsExist(ctx, t, reconcilerTestEnv, etcdInstance, 2*time.Minute, 2*time.Second) + t.Logf("successfully deleted all resources for etcd instance: %s, waiting for finalizer to be removed from etcd...", etcdInstance.Name) + assertETCDFinalizer(t, cl, client.ObjectKeyFromObject(etcdInstance), false, 2*time.Minute, 2*time.Second) +} + +func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T) { + // ********************************** setup ********************************** + testNs := createTestNamespaceName(t) + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + WithDefaultBackup(). + WithAnnotations(map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }). + Build() + // create the test client builder and record errors for delete operations for client service and snapshot lease. + testClientBuilder := testutils.NewTestClientBuilder(). + RecordErrorForObjects(testutils.ClientMethodDelete, testutils.TestAPIInternalErr, client.ObjectKey{Name: etcdInstance.GetClientServiceName(), Namespace: etcdInstance.Namespace}). + RecordErrorForObjectsMatchingLabels(testutils.ClientMethodDeleteAll, etcdInstance.Namespace, map[string]string{ + druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: etcdInstance.Name, + }, testutils.TestAPIInternalErr) + // create the integration test environment with the test client builder + itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnvWithClientBuilder(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}, testClientBuilder) + defer itTestEnvCloser(t) + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) + + // create test namespace + itTestEnv.CreateTestNamespace(testNs) + t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) + + g := NewWithT(t) + ctx := context.Background() + cl := itTestEnv.GetClient() + + // ensure that the backup store is enabled and backup secrets are created + g.Expect(etcdInstance.Spec.Backup.Store).ToNot(BeNil()) + g.Expect(etcdInstance.Spec.Backup.Store.SecretRef).ToNot(BeNil()) + g.Expect(testutils.CreateSecrets(ctx, cl, testNs, etcdInstance.Spec.Backup.Store.SecretRef.Name)).To(Succeed()) + t.Logf("successfully created backup secrets for etcd instance: %s", etcdInstance.Name) + + createEtcdInstanceWithFinalizer(ctx, t, reconcilerTestEnv, etcdInstance) + + // ******************************* test etcd deletion flow ******************************* + // mark etcd for deletion + g.Expect(cl.Delete(ctx, etcdInstance)).To(Succeed()) + t.Logf("successfully marked etcd instance for deletion: %s, waiting for resources to be removed...", etcdInstance.Name) + // assert removal of all components except client service and snapshot lease. + assertComponentsDoNotExist(ctx, t, reconcilerTestEnv, etcdInstance, []operator.Kind{ + operator.MemberLeaseKind, + operator.PeerServiceKind, + operator.ConfigMapKind, + operator.PodDisruptionBudgetKind, + operator.ServiceAccountKind, + operator.RoleKind, + operator.RoleBindingKind, + operator.StatefulSetKind}, 2*time.Minute, 2*time.Second) + assertSelectedComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, []operator.Kind{operator.ClientServiceKind, operator.SnapshotLeaseKind}, 2*time.Minute, 2*time.Second) + // assert that the last operation and last errors are updated correctly. + expectedLastOperation := &druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeDelete, + State: druidv1alpha1.LastOperationStateError, + } + expectedLastErrors := []druidv1alpha1.LastError{ + { + Code: "ERR_DELETE_CLIENT_SERVICE", + }, + { + Code: "ERR_DELETE_SNAPSHOT_LEASE", + }, + } + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, expectedLastErrors, 2*time.Minute, 2*time.Second) + // assert that the finalizer has not been removed as all resources have not been deleted yet. + assertETCDFinalizer(t, cl, client.ObjectKeyFromObject(etcdInstance), true, 10*time.Second, 2*time.Second) +} + +// ------------------------------ reconcile status tests ------------------------------ + +// ------------------------- Helper functions ------------------------- func createTestNamespaceName(t *testing.T) string { b := make([]byte, 4) _, err := rand.Read(b) @@ -150,9 +308,8 @@ func createTestNamespaceName(t *testing.T) string { return fmt.Sprintf("%s-%s", testNamespacePrefix, namespaceSuffix) } -func initializeEtcdReconcilerTestEnv(t *testing.T, autoReconcile bool) ReconcilerTestEnv { +func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTestEnv, autoReconcile bool) ReconcilerTestEnv { g := NewWithT(t) - itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) var ( reconciler *etcd.Reconciler err error @@ -174,8 +331,27 @@ func initializeEtcdReconcilerTestEnv(t *testing.T, autoReconcile bool) Reconcile itTestEnv.StartManager() t.Log("successfully registered etcd reconciler with manager and started manager") return ReconcilerTestEnv{ - itTestEnv: itTestEnv, - itTestEnvCloser: itTestEnvCloser, - reconciler: reconciler, + itTestEnv: itTestEnv, + reconciler: reconciler, } } + +func createEtcdInstanceWithFinalizer(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { + g := NewWithT(t) + cl := reconcilerTestEnv.itTestEnv.GetClient() + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + t.Logf("trigggered creation of etcd instance: %s, waiting for resources to be created...", etcdInstance.Name) + // ascertain that all etcd resources are created + assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 3*time.Minute, 2*time.Second) + t.Logf("successfully created all resources for etcd instance: %s", etcdInstance.Name) + // add finalizer + addFinalizer(ctx, g, cl, client.ObjectKeyFromObject(etcdInstance)) + t.Logf("successfully added finalizer to etcd instance: %s", etcdInstance.Name) +} + +func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey client.ObjectKey) { + etcdInstance := &druidv1alpha1.Etcd{} + g.Expect(cl.Get(ctx, etcdObjectKey, etcdInstance)).To(Succeed()) + g.Expect(controllerutils.AddFinalizers(ctx, cl, etcdInstance, common.FinalizerName)).To(Succeed()) +} diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index 2e146879b..9f9f3763f 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -5,6 +5,7 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -15,6 +16,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -26,7 +28,6 @@ type IntegrationTestEnv interface { RegisterReconciler(addToMgrFn AddToManagerFn) StartManager() GetClient() client.Client - GetConfig() *rest.Config CreateTestNamespace(name string) GetLogger() logr.Logger } @@ -42,9 +43,9 @@ type itTestEnv struct { logger logr.Logger } -type IntegrationTestEnvCloser func() +type IntegrationTestEnvCloser func(t *testing.T) -func NewIntegrationTestEnv(t *testing.T, loggerName string, crdDirectoryPaths []string) (IntegrationTestEnv, IntegrationTestEnvCloser) { +func NewIntegrationTestEnvWithClientBuilder(t *testing.T, loggerName string, crdDirectoryPaths []string, clientBuilder *testutils.TestClientBuilder) (IntegrationTestEnv, IntegrationTestEnvCloser) { ctx, cancelFunc := context.WithCancel(context.Background()) itEnv := &itTestEnv{ ctx: ctx, @@ -54,13 +55,18 @@ func NewIntegrationTestEnv(t *testing.T, loggerName string, crdDirectoryPaths [] } druidScheme := itEnv.prepareScheme() itEnv.createTestEnvironment(druidScheme, crdDirectoryPaths) - itEnv.createManager() - return itEnv, func() { + itEnv.createManager(druidScheme, clientBuilder) + return itEnv, func(t *testing.T) { itEnv.cancelFn() itEnv.g.Expect(itEnv.testEnv.Stop()).To(Succeed()) + t.Log("stopped test environment") } } +func NewIntegrationTestEnv(t *testing.T, loggerName string, crdDirectoryPaths []string) (IntegrationTestEnv, IntegrationTestEnvCloser) { + return NewIntegrationTestEnvWithClientBuilder(t, loggerName, crdDirectoryPaths, testutils.NewTestClientBuilder()) +} + func (t *itTestEnv) RegisterReconciler(addToMgrFn AddToManagerFn) { addToMgrFn(t.mgr) } @@ -75,10 +81,6 @@ func (t *itTestEnv) GetClient() client.Client { return t.client } -func (t *itTestEnv) GetConfig() *rest.Config { - return t.config -} - func (t *itTestEnv) CreateTestNamespace(name string) { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -106,22 +108,33 @@ func (t *itTestEnv) createTestEnvironment(scheme *k8sruntime.Scheme, crdDirector cfg, err := testEnv.Start() t.g.Expect(err).ToNot(HaveOccurred()) - cl, err := client.New(cfg, client.Options{Scheme: scheme}) - t.g.Expect(err).ToNot(HaveOccurred()) t.config = cfg t.testEnv = testEnv - t.client = cl } -func (t *itTestEnv) createManager() { +func (t *itTestEnv) createManager(scheme *k8sruntime.Scheme, clientBuilder *testutils.TestClientBuilder) { mgr, err := manager.New(t.config, manager.Options{ + Scheme: scheme, MetricsBindAddress: "0", ClientDisableCacheFor: []client.Object{ &corev1.Event{}, &eventsv1beta1.Event{}, &eventsv1.Event{}, }, + NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + cl, err := client.New(config, options) + if err != nil { + return nil, err + } + testCl := clientBuilder.WithClient(cl).Build() + return client.NewDelegatingClient(client.NewDelegatingClientInput{ + CacheReader: cache, + Client: testCl, + UncachedObjects: uncachedObjects, + }) + }, }) t.g.Expect(err).ToNot(HaveOccurred()) t.mgr = mgr + t.client = mgr.GetClient() } diff --git a/test/utils/client.go b/test/utils/client.go new file mode 100644 index 000000000..f7568d679 --- /dev/null +++ b/test/utils/client.go @@ -0,0 +1,235 @@ +package utils + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// ClientMethod is a name of the method on client.Client for which an error is recorded. +type ClientMethod string + +const ( + // ClientMethodGet is the name of the Get method on client.Client. + ClientMethodGet ClientMethod = "Get" + // ClientMethodList is the name of the List method on client.Client. + ClientMethodList ClientMethod = "List" + // ClientMethodCreate is the name of the Create method on client.Client. + ClientMethodCreate ClientMethod = "Create" + // ClientMethodDelete is the name of the Delete method on client.Client. + ClientMethodDelete ClientMethod = "Delete" + // ClientMethodDeleteAll is the name of the DeleteAllOf method on client.Client. + ClientMethodDeleteAll ClientMethod = "DeleteAll" + // ClientMethodPatch is the name of the Patch method on client.Client. + ClientMethodPatch ClientMethod = "Patch" + // ClientMethodUpdate is the name of the Update method on client.Client. + ClientMethodUpdate ClientMethod = "Update" +) + +// errorRecord contains the recorded error for a specific client.Client method and identifiers such as name, namespace and matching labels. +type errorRecord struct { + method ClientMethod + resourceName string + resourceNamespace string + labels labels.Set + err error +} + +// TestClientBuilder builds a client.Client which will also react to the configured errors. +type TestClientBuilder struct { + delegatingClient client.Client + errorRecords []errorRecord +} + +// CreateTestFakeClientForObjects is a convenience function which creates a test client which uses a fake client as a delegate and reacts to the configured errors for the given object key. +func CreateTestFakeClientForObjects(getErr, createErr, patchErr, deleteErr *apierrors.StatusError, existingObjects []client.Object, objKeys ...client.ObjectKey) client.Client { + fakeDelegateClientBuilder := fake.NewClientBuilder() + if existingObjects != nil && len(existingObjects) > 0 { + fakeDelegateClientBuilder.WithObjects(existingObjects...) + } + fakeDelegateClient := fakeDelegateClientBuilder.Build() + testClientBuilder := NewTestClientBuilder().WithClient(fakeDelegateClient) + for _, objKey := range objKeys { + testClientBuilder.RecordErrorForObjects(ClientMethodGet, getErr, objKey). + RecordErrorForObjects(ClientMethodCreate, createErr, objKey). + RecordErrorForObjects(ClientMethodDelete, deleteErr, objKey). + RecordErrorForObjects(ClientMethodPatch, patchErr, objKey) + } + return testClientBuilder.Build() +} + +// CreateTestFakeClientForAllObjectsInNamespace is a convenience function which creates a test client which uses a fake client as a delegate and reacts to the configured errors for all objects in the given namespace matching the given labels. +func CreateTestFakeClientForAllObjectsInNamespace(deleteAllErr, listErr *apierrors.StatusError, namespace string, matchingLabels map[string]string, existingObjects ...client.Object) client.Client { + fakeDelegateClientBuilder := fake.NewClientBuilder() + if existingObjects != nil && len(existingObjects) > 0 { + fakeDelegateClientBuilder.WithObjects(existingObjects...) + } + fakeDelegateClient := fakeDelegateClientBuilder.Build() + cl := NewTestClientBuilder(). + WithClient(fakeDelegateClient). + RecordErrorForObjectsMatchingLabels(ClientMethodDeleteAll, namespace, matchingLabels, deleteAllErr). + RecordErrorForObjectsMatchingLabels(ClientMethodList, namespace, matchingLabels, listErr). + Build() + return cl +} + +// NewTestClientBuilder creates a new instance of TestClientBuilder. +func NewTestClientBuilder() *TestClientBuilder { + return &TestClientBuilder{} +} + +// WithClient sets the client.Client to be used as a delegate for the test client. +func (b *TestClientBuilder) WithClient(client client.Client) *TestClientBuilder { + b.delegatingClient = client + return b +} + +// RecordErrorForObjects records an error for a specific client.Client method and object keys. +func (b *TestClientBuilder) RecordErrorForObjects(method ClientMethod, err *apierrors.StatusError, objectKeys ...client.ObjectKey) *TestClientBuilder { + // this method records error, so if nil error is passed then there is no need to create any error record. + if err == nil { + return b + } + for _, objectKey := range objectKeys { + b.errorRecords = append(b.errorRecords, errorRecord{ + method: method, + resourceName: objectKey.Name, + resourceNamespace: objectKey.Namespace, + err: err, + }) + } + return b +} + +// RecordErrorForObjectsMatchingLabels records an error for a specific client.Client method and objects in a given namespace matching the given labels. +func (b *TestClientBuilder) RecordErrorForObjectsMatchingLabels(method ClientMethod, namespace string, matchingLabels map[string]string, err *apierrors.StatusError) *TestClientBuilder { + // this method records error, so if nil error is passed then there is no need to create any error record. + if err == nil { + return b + } + b.errorRecords = append(b.errorRecords, errorRecord{ + method: method, + resourceNamespace: namespace, + labels: createLabelSet(matchingLabels), + err: err, + }) + return b +} + +// Build creates a new instance of client.Client which will react to the configured errors. +func (b *TestClientBuilder) Build() client.Client { + return &testClient{ + delegate: b.delegatingClient, + errorRecords: b.errorRecords, + } +} + +// testClient is a client.Client implementation which reacts to the configured errors. +type testClient struct { + delegate client.Client + errorRecords []errorRecord +} + +// ---------------------------------- Implementation of client.Client ---------------------------------- + +func (c *testClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.getRecordedObjectError(ClientMethodGet, key); err != nil { + return err + } + return c.delegate.Get(ctx, key, obj, opts...) +} + +func (c *testClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + if err := c.getRecordObjectCollectionError(ClientMethodList, listOpts.Namespace, listOpts.LabelSelector); err != nil { + return err + } + return c.delegate.List(ctx, list, opts...) +} + +func (c *testClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.getRecordedObjectError(ClientMethodCreate, client.ObjectKeyFromObject(obj)); err != nil { + return err + } + return c.delegate.Create(ctx, obj, opts...) +} + +func (c *testClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if err := c.getRecordedObjectError(ClientMethodDelete, client.ObjectKeyFromObject(obj)); err != nil { + return err + } + return c.delegate.Delete(ctx, obj, opts...) +} + +func (c *testClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + deleteOpts := client.DeleteAllOfOptions{} + deleteOpts.ApplyOptions(opts) + if err := c.getRecordObjectCollectionError(ClientMethodDeleteAll, deleteOpts.Namespace, deleteOpts.LabelSelector); err != nil { + return err + } + return c.delegate.DeleteAllOf(ctx, obj, opts...) +} + +func (c *testClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := c.getRecordedObjectError(ClientMethodPatch, client.ObjectKeyFromObject(obj)); err != nil { + return err + } + return c.delegate.Patch(ctx, obj, patch, opts...) +} + +func (c *testClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if err := c.getRecordedObjectError(ClientMethodUpdate, client.ObjectKeyFromObject(obj)); err != nil { + return err + } + return c.delegate.Update(ctx, obj, opts...) +} + +func (c *testClient) Status() client.SubResourceWriter { + return c.delegate.Status() +} + +func (c *testClient) SubResource(subResource string) client.SubResourceClient { + return c.delegate.SubResource(subResource) +} + +func (c *testClient) Scheme() *runtime.Scheme { + return c.delegate.Scheme() +} + +func (c *testClient) RESTMapper() meta.RESTMapper { + return c.delegate.RESTMapper() +} + +// ---------------------------------- Helper methods ---------------------------------- +func (c *testClient) getRecordedObjectError(method ClientMethod, objKey client.ObjectKey) error { + for _, errRecord := range c.errorRecords { + recordedObjKey := client.ObjectKey{Name: errRecord.resourceName, Namespace: errRecord.resourceNamespace} + if errRecord.method == method && recordedObjKey == objKey { + return errRecord.err + } + } + return nil +} + +func (c *testClient) getRecordObjectCollectionError(method ClientMethod, namespace string, labelSelector labels.Selector) error { + for _, errRecord := range c.errorRecords { + if errRecord.method == method && errRecord.resourceNamespace == namespace && labelSelector.Matches(errRecord.labels) { + return errRecord.err + } + } + return nil +} + +func createLabelSet(l map[string]string) labels.Set { + s := labels.Set{} + for k, v := range l { + s[k] = v + } + return s +} diff --git a/test/utils/fakeclient.go b/test/utils/fakeclient.go deleted file mode 100644 index b170e73ce..000000000 --- a/test/utils/fakeclient.go +++ /dev/null @@ -1,135 +0,0 @@ -package utils - -import ( - "context" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -// FakeClientBuilder builds a fake client with initial set of objects and additionally -// provides an ability to set custom errors for operations supported on the client. -type FakeClientBuilder struct { - clientBuilder *fakeclient.ClientBuilder - getErr *apierrors.StatusError - listErr *apierrors.StatusError - createErr *apierrors.StatusError - patchErr *apierrors.StatusError - deleteErr *apierrors.StatusError - deleteAllOfErr *apierrors.StatusError -} - -// NewFakeClientBuilder creates a new FakeClientBuilder. -func NewFakeClientBuilder() *FakeClientBuilder { - return &FakeClientBuilder{ - clientBuilder: fakeclient.NewClientBuilder(), - } -} - -// WithObjects initializes the underline controller-runtime fake client with objects. -func (b *FakeClientBuilder) WithObjects(objs ...client.Object) *FakeClientBuilder { - b.clientBuilder.WithObjects(objs...) - return b -} - -// WithGetError sets the error that should be returned when a Get request is made on the fake client. -func (b *FakeClientBuilder) WithGetError(err *apierrors.StatusError) *FakeClientBuilder { - b.getErr = err - return b -} - -func (b *FakeClientBuilder) WithListError(err *apierrors.StatusError) *FakeClientBuilder { - b.listErr = err - return b -} - -// WithCreateError sets the error that should be returned when a Create request is made on the fake client. -func (b *FakeClientBuilder) WithCreateError(err *apierrors.StatusError) *FakeClientBuilder { - b.createErr = err - return b -} - -// WithPatchError sets the error that should be returned when a Patch request is made on the fake client. -func (b *FakeClientBuilder) WithPatchError(err *apierrors.StatusError) *FakeClientBuilder { - b.patchErr = err - return b -} - -// WithDeleteError sets the error that should be returned when a Delete request is made on the fake client. -func (b *FakeClientBuilder) WithDeleteError(err *apierrors.StatusError) *FakeClientBuilder { - b.deleteErr = err - return b -} - -func (b *FakeClientBuilder) WithDeleteAllOfError(err *apierrors.StatusError) *FakeClientBuilder { - b.deleteAllOfErr = err - return b -} - -// Build returns an instance of client.WithWatch which has capability to return the configured errors for operations. -func (b *FakeClientBuilder) Build() client.WithWatch { - return &fakeClient{ - WithWatch: b.clientBuilder.Build(), - getErr: b.getErr, - listErr: b.listErr, - createErr: b.createErr, - patchErr: b.patchErr, - deleteErr: b.deleteErr, - deleteAllOfErr: b.deleteAllOfErr, - } -} - -type fakeClient struct { - client.WithWatch - getErr *apierrors.StatusError - listErr *apierrors.StatusError - createErr *apierrors.StatusError - patchErr *apierrors.StatusError - deleteErr *apierrors.StatusError - deleteAllOfErr *apierrors.StatusError -} - -// Get overwrites the fake client Get implementation with a capability to return any configured error. -func (f *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - if f.getErr != nil { - return f.getErr - } - return f.WithWatch.Get(ctx, key, obj, opts...) -} - -func (f *fakeClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - if f.listErr != nil { - return f.listErr - } - return f.WithWatch.List(ctx, list, opts...) -} - -// Delete overwrites the fake client Get implementation with a capability to return any configured error. -func (f *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - if f.deleteErr != nil { - return f.deleteErr - } - return f.WithWatch.Delete(ctx, obj, opts...) -} - -func (f *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { - if f.deleteAllOfErr != nil { - return f.deleteAllOfErr - } - return f.WithWatch.DeleteAllOf(ctx, obj, opts...) -} - -func (f *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - if f.patchErr != nil { - return f.patchErr - } - return f.WithWatch.Patch(ctx, obj, patch, opts...) -} - -func (f *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - if f.createErr != nil { - return f.createErr - } - return f.WithWatch.Create(ctx, obj, opts...) -} From 17da300a0c978ad8cf62475e0394d44ed5795841 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 9 Feb 2024 18:26:08 +0530 Subject: [PATCH 086/235] Changes to sentinel webhook --- .../templates/controller-deployment.yaml | 1 + internal/controller/etcd/reconcile_spec.go | 22 ++++--------------- internal/utils/etcd.go | 19 ++++++++++++++++ internal/webhook/sentinel/config.go | 15 ++++++++++--- internal/webhook/sentinel/handler.go | 22 +++++++++++++++---- 5 files changed, 54 insertions(+), 25 deletions(-) create mode 100644 internal/utils/etcd.go diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index 51243fee9..669f786f2 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -56,6 +56,7 @@ spec: {{- if .Values.enableSentinelWebhook }} - --enable-sentinel-webhook={{ .Values.enableSentinelWebhook }} + - --reconciler-service-account=system:serviceaccount:{{ .Release.Namespace }}:etcd-druid {{- end }} resources: diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 0cefee2a0..6b6161cbe 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -16,14 +16,12 @@ package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/controller/utils" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/component" + druidutils "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -55,7 +53,7 @@ func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, et withOpAnnotation := etcd.DeepCopy() delete(etcd.Annotations, v1beta1constants.GardenerOperation) if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { - return utils.ReconcileWithError(err) + return ctrlutils.ReconcileWithError(err) } } return ctrlutils.ContinueReconcile() @@ -69,7 +67,7 @@ func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey for _, kind := range resourceOperators { op := r.operatorRegistry.GetOperator(kind) if err := op.Sync(ctx, etcd); err != nil { - return utils.ReconcileWithError(err) + return ctrlutils.ReconcileWithError(err) } } return ctrlutils.ContinueReconcile() @@ -123,7 +121,7 @@ func (r *Reconciler) recordIncompleteReconcileOperation(ctx component.OperatorCo // - Reconciliation is not initiated if EnableEtcdSpecAutoReconcile is false and none of the relevant annotations are present. func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { // Check if spec reconciliation has been suspended, if yes, then record the event and return false. - if suspendReconcileAnnotKey := r.getSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil { + if suspendReconcileAnnotKey := druidutils.GetSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil { r.recordEtcdSpecReconcileSuspension(etcd, *suspendReconcileAnnotKey) return false } @@ -147,18 +145,6 @@ func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { return false } -// getSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent -// to suspend spec reconciliation for this etcd resource. If no annotation is set then it will return nil. -func (r *Reconciler) getSuspendEtcdSpecReconcileAnnotationKey(etcd *druidv1alpha1.Etcd) *string { - var annotationKey *string - if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) { - annotationKey = pointer.String(druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) - } else if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.IgnoreReconciliationAnnotation) { - annotationKey = pointer.String(druidv1alpha1.IgnoreReconciliationAnnotation) - } - return annotationKey -} - func (r *Reconciler) recordEtcdSpecReconcileSuspension(etcd *druidv1alpha1.Etcd, annotationKey string) { r.recorder.Eventf( etcd, diff --git a/internal/utils/etcd.go b/internal/utils/etcd.go new file mode 100644 index 000000000..692ff8f94 --- /dev/null +++ b/internal/utils/etcd.go @@ -0,0 +1,19 @@ +package utils + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent +// to suspend spec reconciliation for this etcd resource. If no annotation is set then it will return nil. +func GetSuspendEtcdSpecReconcileAnnotationKey(etcd *druidv1alpha1.Etcd) *string { + var annotationKey *string + if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) { + annotationKey = pointer.String(druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) + } else if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.IgnoreReconciliationAnnotation) { + annotationKey = pointer.String(druidv1alpha1.IgnoreReconciliationAnnotation) + } + return annotationKey +} diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/sentinel/config.go index def8746a3..f4c9eebfa 100644 --- a/internal/webhook/sentinel/config.go +++ b/internal/webhook/sentinel/config.go @@ -1,21 +1,30 @@ package sentinel -import flag "github.com/spf13/pflag" +import ( + "fmt" + flag "github.com/spf13/pflag" +) const ( - enableSentinelWebhookFlagName = "enable-sentinel-webhook" + enableSentinelWebhookFlagName = "enable-sentinel-webhook" + reconcilerServiceAccountFlagName = "reconciler-service-account" - defaultEnableSentinelWebhook = false + defaultEnableSentinelWebhook = false + defaultReconcilerServiceAccount = "system:serviceaccount:default:etcd-druid" ) // Config defines the configuration for the Sentinel Webhook. type Config struct { // Enabled indicates whether the Sentinel Webhook is enabled. Enabled bool + // ReconcilerServiceAccount is the name of the service account used by etcd-druid for reconciling etcd resources. + ReconcilerServiceAccount string } // InitFromFlags initializes the config from the provided CLI flag set. func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.BoolVar(&cfg.Enabled, enableSentinelWebhookFlagName, defaultEnableSentinelWebhook, "Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid.") + fs.StringVar(&cfg.ReconcilerServiceAccount, reconcilerServiceAccountFlagName, defaultReconcilerServiceAccount, + fmt.Sprintf("The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. Default: %s", defaultReconcilerServiceAccount)) } diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 82b792588..78ce1f691 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -6,6 +6,7 @@ import ( "net/http" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + druidutils "github.com/gardener/etcd-druid/internal/utils" "github.com/go-logr/logr" admissionv1 "k8s.io/api/admission/v1" @@ -56,7 +57,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R err error ) - log := h.logger.WithValues("resourceKind", req.Kind.Kind, "name", req.Name, "namespace", req.Namespace, "operation", req.Operation) + log := h.logger.WithValues("resourceKind", req.Kind.Kind, "name", req.Name, "namespace", req.Namespace, "operation", req.Operation, "user", req.UserInfo.Username) log.Info("Sentinel webhook invoked") if req.Operation == admissionv1.Delete { @@ -102,17 +103,30 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } - // allow operations on resources if any etcd operation is currently under processing - if etcd.Status.LastOperation != nil && etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateProcessing { - return admission.Allowed(fmt.Sprintf("ongoing processing of etcd %s by etcd-druid requires changes to resources", etcd.Name)) + // allow changes to resources if etcd spec reconciliation is currently suspended + if suspendReconcileAnnotKey := druidutils.GetSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil && metav1.HasAnnotation(etcd.ObjectMeta, *suspendReconcileAnnotKey) { + return admission.Allowed(fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", etcd.Name)) } + // allow changes to resources if etcd has annotation druid.gardener.cloud/resource-protection: false if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.ResourceProtectionAnnotation) { if etcd.GetAnnotations()[druidv1alpha1.ResourceProtectionAnnotation] == "false" { return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) } } + // allow operations on resources if any etcd operation is currently under processing, but only by etcd-druid + if etcd.Status.LastOperation != nil && + (etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateProcessing || + etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateError) { + if req.UserInfo.Username == h.config.ReconcilerServiceAccount { + return admission.Allowed(fmt.Sprintf("ongoing processing of etcd %s by etcd-druid requires changes to resources", etcd.Name)) + } + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing processing of etcd %s by etcd-druid", etcd.Name)) + } + // TODO: how to make VPA work for sts updates? + // TODO: add unit tests for Handle() + return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) } From 2200194c33ba5ce483c4ef8b4bc30a283cd03489 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 13 Feb 2024 22:24:23 +0530 Subject: [PATCH 087/235] Add etcd helper functions, along with unit tests --- api/v1alpha1/types_etcd.go | 30 ++++ api/v1alpha1/types_etcd_test.go | 154 ++++++++++++++++++++- internal/controller/etcd/reconcile_spec.go | 3 +- internal/utils/etcd.go | 19 --- 4 files changed, 183 insertions(+), 23 deletions(-) delete mode 100644 internal/utils/etcd.go diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 0818835ad..b62e84bc3 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -576,3 +576,33 @@ func (e *Etcd) IsBackupStoreEnabled() bool { func (e *Etcd) IsMarkedForDeletion() bool { return !e.DeletionTimestamp.IsZero() } + +// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent +// to suspend spec reconciliation for this etcd resource. If no annotation is set then it will return nil. +func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { + if metav1.HasAnnotation(e.ObjectMeta, SuspendEtcdSpecReconcileAnnotation) { + return pointer.String(SuspendEtcdSpecReconcileAnnotation) + } + if metav1.HasAnnotation(e.ObjectMeta, IgnoreReconciliationAnnotation) { + return pointer.String(IgnoreReconciliationAnnotation) + } + return nil +} + +func (e *Etcd) IsReconciliationSuspended() bool { + suspendReconcileAnnotKey := e.GetSuspendEtcdSpecReconcileAnnotationKey() + return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) +} + +func (e *Etcd) AreManagedResourcesProtected() bool { + if metav1.HasAnnotation(e.ObjectMeta, ResourceProtectionAnnotation) { + return e.GetAnnotations()[ResourceProtectionAnnotation] != "false" + } + return true +} + +func (e *Etcd) IsBeingProcessed() bool { + return e.Status.LastOperation != nil && + (e.Status.LastOperation.State == LastOperationStateProcessing || + e.Status.LastOperation.State == LastOperationStateError) +} diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 47a468f47..7f47af0f7 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -108,8 +108,8 @@ var _ = Describe("Etcd", func() { Context("GetDefaultLabels", func() { It("should return the default labels for etcd", func() { expected := map[string]string{ - "name": "etcd", - "instance": "foo", + LabelManagedByKey: LabelManagedByValue, + LabelPartOfKey: "foo", } Expect(created.GetDefaultLabels()).To(Equal(expected)) }) @@ -140,6 +140,156 @@ var _ = Describe("Etcd", func() { Expect(created.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) }) }) + + Context("IsBackupStoreEnabled", func() { + Context("when backup is enabled", func() { + It("should return true", func() { + Expect(created.IsBackupStoreEnabled()).To(Equal(true)) + }) + }) + Context("when backup is not enabled", func() { + It("should return false", func() { + created.Spec.Backup = BackupSpec{} + Expect(created.IsBackupStoreEnabled()).To(Equal(false)) + }) + }) + }) + + Context("IsMarkedForDeletion", func() { + Context("when deletion timestamp is not set", func() { + It("should return false", func() { + Expect(created.IsMarkedForDeletion()).To(Equal(false)) + }) + }) + Context("when deletion timestamp is set", func() { + It("should return true", func() { + created.DeletionTimestamp = &metav1.Time{Time: time.Now()} + Expect(created.IsMarkedForDeletion()).To(Equal(true)) + }) + }) + }) + + Context("GetSuspendEtcdSpecReconcileAnnotationKey", func() { + ignoreReconciliationAnnotation := IgnoreReconciliationAnnotation + suspendEtcdSpecReconcileAnnotation := SuspendEtcdSpecReconcileAnnotation + Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { + It("should return druid.gardener.cloud/ignore-reconciliation", func() { + created.Annotations = map[string]string{ + IgnoreReconciliationAnnotation: "true", + } + Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&ignoreReconciliationAnnotation)) + }) + }) + Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { + It("should return druid.gardener.cloud/suspend-etcd-spec-reconcile", func() { + created.Annotations = map[string]string{ + SuspendEtcdSpecReconcileAnnotation: "true", + } + Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) + }) + }) + Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { + It("should return druid.gardener.cloud/suspend-etcd-spec-reconcile", func() { + created.Annotations = map[string]string{ + SuspendEtcdSpecReconcileAnnotation: "true", + IgnoreReconciliationAnnotation: "true", + } + Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) + }) + }) + Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { + It("should return nil string pointer", func() { + var nilString *string + Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(nilString)) + }) + }) + }) + + Context("IsReconciliationSuspended", func() { + Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { + It("should return true", func() { + created.Annotations = map[string]string{ + SuspendEtcdSpecReconcileAnnotation: "true", + } + Expect(created.IsReconciliationSuspended()).To(Equal(true)) + }) + }) + Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { + It("should return true", func() { + created.Annotations = map[string]string{ + IgnoreReconciliationAnnotation: "true", + } + Expect(created.IsReconciliationSuspended()).To(Equal(true)) + }) + }) + Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { + It("should return true", func() { + created.Annotations = map[string]string{ + SuspendEtcdSpecReconcileAnnotation: "true", + IgnoreReconciliationAnnotation: "true", + } + Expect(created.IsReconciliationSuspended()).To(Equal(true)) + }) + }) + Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { + It("should return false", func() { + Expect(created.IsReconciliationSuspended()).To(Equal(false)) + }) + }) + }) + + Context("AreManagedResourcesProtected", func() { + Context("when etcd has annotation druid.gardener.cloud/resource-protection: false", func() { + It("should return false", func() { + created.Annotations = map[string]string{ + ResourceProtectionAnnotation: "false", + } + Expect(created.AreManagedResourcesProtected()).To(Equal(false)) + }) + }) + Context("when etcd has annotation druid.gardener.cloud/resource-protection: true", func() { + It("should return true", func() { + Expect(created.AreManagedResourcesProtected()).To(Equal(true)) + }) + }) + Context("when etcd does not have annotation druid.gardener.cloud/resource-protection set", func() { + It("should return true", func() { + Expect(created.AreManagedResourcesProtected()).To(Equal(true)) + }) + }) + }) + + Context("IsBeingProcessed", func() { + Context("when etcd status has lastOperation and its state is Processing", func() { + It("should return true", func() { + created.Status.LastOperation = &LastOperation{ + State: LastOperationStateProcessing, + } + Expect(created.IsBeingProcessed()).To(Equal(true)) + }) + }) + Context("when etcd status has lastOperation and its state is Error", func() { + It("should return true", func() { + created.Status.LastOperation = &LastOperation{ + State: LastOperationStateError, + } + Expect(created.IsBeingProcessed()).To(Equal(true)) + }) + }) + Context("when etcd status has lastOperation and its state is neither Processing or Error", func() { + It("should return false", func() { + created.Status.LastOperation = &LastOperation{ + State: LastOperationStateSucceeded, + } + Expect(created.IsBeingProcessed()).To(Equal(false)) + }) + }) + Context("when etcd status does not have lastOperation populated", func() { + It("should return false", func() { + Expect(created.IsBeingProcessed()).To(Equal(false)) + }) + }) + }) }) func getEtcd(name, namespace string) *Etcd { diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 6b6161cbe..c26fa2872 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -19,7 +19,6 @@ import ( ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/component" - druidutils "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -121,7 +120,7 @@ func (r *Reconciler) recordIncompleteReconcileOperation(ctx component.OperatorCo // - Reconciliation is not initiated if EnableEtcdSpecAutoReconcile is false and none of the relevant annotations are present. func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { // Check if spec reconciliation has been suspended, if yes, then record the event and return false. - if suspendReconcileAnnotKey := druidutils.GetSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil { + if suspendReconcileAnnotKey := etcd.GetSuspendEtcdSpecReconcileAnnotationKey(); suspendReconcileAnnotKey != nil { r.recordEtcdSpecReconcileSuspension(etcd, *suspendReconcileAnnotKey) return false } diff --git a/internal/utils/etcd.go b/internal/utils/etcd.go deleted file mode 100644 index 692ff8f94..000000000 --- a/internal/utils/etcd.go +++ /dev/null @@ -1,19 +0,0 @@ -package utils - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" -) - -// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent -// to suspend spec reconciliation for this etcd resource. If no annotation is set then it will return nil. -func GetSuspendEtcdSpecReconcileAnnotationKey(etcd *druidv1alpha1.Etcd) *string { - var annotationKey *string - if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) { - annotationKey = pointer.String(druidv1alpha1.SuspendEtcdSpecReconcileAnnotation) - } else if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.IgnoreReconciliationAnnotation) { - annotationKey = pointer.String(druidv1alpha1.IgnoreReconciliationAnnotation) - } - return annotationKey -} From db1301833d8df987d763653875c0314e31f33bfd Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 14 Feb 2024 14:09:50 +0530 Subject: [PATCH 088/235] Add exemptServiceAccounts config for sentinel webhook --- .../templates/controller-deployment.yaml | 7 ++++-- charts/druid/values.yaml | 4 ++- internal/webhook/sentinel/config.go | 9 +++++++ internal/webhook/sentinel/handler.go | 25 +++++++++---------- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index 669f786f2..b42ccb3cf 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -54,9 +54,12 @@ spec: - --webhook-server-tls-server-cert-dir={{ .Values.server.webhook.tls.serverCertDir }} {{- end }} - {{- if .Values.enableSentinelWebhook }} - - --enable-sentinel-webhook={{ .Values.enableSentinelWebhook }} + {{- if (.Values.sentinelWebhook).enabled }} + - --enable-sentinel-webhook=true - --reconciler-service-account=system:serviceaccount:{{ .Release.Namespace }}:etcd-druid + {{- if .Values.sentinelWebhook.exemptServiceAccounts }} + - --sentinel-exempt-service-accounts={{ join "," .Values.sentinelWebhook.exemptServiceAccounts }} + {{- end }} {{- end }} resources: diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index c50f523a5..3fb828c2f 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -19,4 +19,6 @@ server: tls: serverCertDir: /etc/webhook-server-tls -enableSentinelWebhook: true +sentinelWebhook: + enabled: true + exemptServiceAccounts: {} diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/sentinel/config.go index f4c9eebfa..c6fbac0cd 100644 --- a/internal/webhook/sentinel/config.go +++ b/internal/webhook/sentinel/config.go @@ -8,17 +8,24 @@ import ( const ( enableSentinelWebhookFlagName = "enable-sentinel-webhook" reconcilerServiceAccountFlagName = "reconciler-service-account" + exemptServiceAccountsFlagName = "sentinel-exempt-service-accounts" defaultEnableSentinelWebhook = false defaultReconcilerServiceAccount = "system:serviceaccount:default:etcd-druid" ) +var ( + defaultExemptServiceAccounts []string +) + // Config defines the configuration for the Sentinel Webhook. type Config struct { // Enabled indicates whether the Sentinel Webhook is enabled. Enabled bool // ReconcilerServiceAccount is the name of the service account used by etcd-druid for reconciling etcd resources. ReconcilerServiceAccount string + // ExemptServiceAccounts is a list of service accounts that are exempt from Sentinel Webhook checks. + ExemptServiceAccounts []string } // InitFromFlags initializes the config from the provided CLI flag set. @@ -27,4 +34,6 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { "Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid.") fs.StringVar(&cfg.ReconcilerServiceAccount, reconcilerServiceAccountFlagName, defaultReconcilerServiceAccount, fmt.Sprintf("The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. Default: %s", defaultReconcilerServiceAccount)) + fs.StringSliceVar(&cfg.ExemptServiceAccounts, exemptServiceAccountsFlagName, defaultExemptServiceAccounts, + "The comma-separated list of fully qualified names of service accounts that are exempt from Sentinel Webhook checks.") } diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 78ce1f691..99ce47424 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -6,7 +6,6 @@ import ( "net/http" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidutils "github.com/gardener/etcd-druid/internal/utils" "github.com/go-logr/logr" admissionv1 "k8s.io/api/admission/v1" @@ -17,7 +16,6 @@ import ( policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -104,28 +102,29 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R } // allow changes to resources if etcd spec reconciliation is currently suspended - if suspendReconcileAnnotKey := druidutils.GetSuspendEtcdSpecReconcileAnnotationKey(etcd); suspendReconcileAnnotKey != nil && metav1.HasAnnotation(etcd.ObjectMeta, *suspendReconcileAnnotKey) { + if etcd.IsReconciliationSuspended() { return admission.Allowed(fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", etcd.Name)) } // allow changes to resources if etcd has annotation druid.gardener.cloud/resource-protection: false - if metav1.HasAnnotation(etcd.ObjectMeta, druidv1alpha1.ResourceProtectionAnnotation) { - if etcd.GetAnnotations()[druidv1alpha1.ResourceProtectionAnnotation] == "false" { - return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) - } + if !etcd.AreManagedResourcesProtected() { + return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) } - // allow operations on resources if any etcd operation is currently under processing, but only by etcd-druid - if etcd.Status.LastOperation != nil && - (etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateProcessing || - etcd.Status.LastOperation.State == druidv1alpha1.LastOperationStateError) { + // allow operations on resources if any etcd operation is currently under processing, but only by etcd-druid, + // and allow exempt service accounts to make changes to resources, but only if etcd is not currently under processing. + if etcd.IsBeingProcessed() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { return admission.Allowed(fmt.Sprintf("ongoing processing of etcd %s by etcd-druid requires changes to resources", etcd.Name)) } return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing processing of etcd %s by etcd-druid", etcd.Name)) + } else { + for _, sa := range h.config.ExemptServiceAccounts { + if req.UserInfo.Username == sa { + return admission.Allowed(fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) + } + } } - // TODO: how to make VPA work for sts updates? - // TODO: add unit tests for Handle() return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) } From 35c515a44276919354acc5ff891975c9b9074f96 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 14 Feb 2024 21:29:59 +0530 Subject: [PATCH 089/235] Clean up api/v1alpha1 unit tests --- api/v1alpha1/suite_test.go | 35 ---------- api/v1alpha1/types_etcd_test.go | 116 +++++++++++++------------------- 2 files changed, 45 insertions(+), 106 deletions(-) diff --git a/api/v1alpha1/suite_test.go b/api/v1alpha1/suite_test.go index 8bca4186f..b680d367a 100644 --- a/api/v1alpha1/suite_test.go +++ b/api/v1alpha1/suite_test.go @@ -5,30 +5,17 @@ package v1alpha1_test import ( - "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - . "github.com/gardener/etcd-druid/api/v1alpha1" ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -var ( - cfg *rest.Config - k8sClient client.Client - testEnv *envtest.Environment -) - func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -40,26 +27,4 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - } - - err := SchemeBuilder.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - cfg, err = testEnv.Start() - Expect(err).ToNot(HaveOccurred()) - Expect(cfg).ToNot(BeNil()) - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).ToNot(HaveOccurred()) - Expect(k8sClient).ToNot(BeNil()) -}) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).ToNot(HaveOccurred()) }) diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 7f47af0f7..ee21234d0 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -5,7 +5,6 @@ package v1alpha1_test import ( - "context" "time" testutils "github.com/gardener/etcd-druid/test/utils" @@ -15,7 +14,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" . "github.com/gardener/etcd-druid/api/v1alpha1" @@ -26,82 +24,58 @@ import ( var _ = Describe("Etcd", func() { var ( - key types.NamespacedName - created, fetched *Etcd + etcd *Etcd ) BeforeEach(func() { - created = getEtcd("foo", "default") - }) - - Context("Create API", func() { - - It("should create an object successfully", func() { - - key = types.NamespacedName{ - Name: "foo", - Namespace: "default", - } - - By("creating an API obj") - Expect(k8sClient.Create(context.Background(), created)).To(Succeed()) - - fetched = &Etcd{} - Expect(k8sClient.Get(context.Background(), key, fetched)).To(Succeed()) - Expect(fetched).To(Equal(created)) - - By("deleting the created object") - Expect(k8sClient.Delete(context.Background(), created)).To(Succeed()) - Expect(k8sClient.Get(context.Background(), key, created)).ToNot(Succeed()) - }) - + etcd = getEtcd("foo", "default") }) Context("GetPeerServiceName", func() { It("should return the correct peer service name", func() { - Expect(created.GetPeerServiceName()).To(Equal("foo-peer")) + Expect(etcd.GetPeerServiceName()).To(Equal("foo-peer")) }) }) Context("GetClientServiceName", func() { It("should return the correct client service name", func() { - Expect(created.GetClientServiceName()).To(Equal("foo-client")) + Expect(etcd.GetClientServiceName()).To(Equal("foo-client")) }) }) Context("GetServiceAccountName", func() { It("should return the correct service account name", func() { - Expect(created.GetServiceAccountName()).To(Equal("foo")) + Expect(etcd.GetServiceAccountName()).To(Equal("foo")) }) }) Context("GetConfigmapName", func() { It("should return the correct configmap name", func() { - Expect(created.GetConfigMapName()).To(Equal("etcd-bootstrap-123456")) + Expect(etcd.GetConfigMapName()).To(Equal("etcd-bootstrap-123456")) }) }) Context("GetCompactionJobName", func() { It("should return the correct compaction job name", func() { - Expect(created.GetCompactionJobName()).To(Equal("foo-compactor")) + Expect(etcd.GetCompactionJobName()).To(Equal("foo-compactor")) }) }) Context("GetOrdinalPodName", func() { It("should return the correct ordinal pod name", func() { - Expect(created.GetOrdinalPodName(0)).To(Equal("foo-0")) + Expect(etcd.GetOrdinalPodName(0)).To(Equal("foo-0")) }) }) Context("GetDeltaSnapshotLeaseName", func() { It("should return the correct delta snapshot lease name", func() { - Expect(created.GetDeltaSnapshotLeaseName()).To(Equal("foo-delta-snap")) + Expect(etcd.GetDeltaSnapshotLeaseName()).To(Equal("foo-delta-snap")) }) }) Context("GetFullSnapshotLeaseName", func() { It("should return the correct full snapshot lease name", func() { - Expect(created.GetFullSnapshotLeaseName()).To(Equal("foo-full-snap")) + Expect(etcd.GetFullSnapshotLeaseName()).To(Equal("foo-full-snap")) }) }) @@ -111,7 +85,7 @@ var _ = Describe("Etcd", func() { LabelManagedByKey: LabelManagedByValue, LabelPartOfKey: "foo", } - Expect(created.GetDefaultLabels()).To(Equal(expected)) + Expect(etcd.GetDefaultLabels()).To(Equal(expected)) }) }) @@ -125,32 +99,32 @@ var _ = Describe("Etcd", func() { Controller: pointer.Bool(true), BlockOwnerDeletion: pointer.Bool(true), } - Expect(created.GetAsOwnerReference()).To(Equal(expected)) + Expect(etcd.GetAsOwnerReference()).To(Equal(expected)) }) }) Context("GetRoleName", func() { It("should return the role name for the Etcd", func() { - Expect(created.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) + Expect(etcd.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) }) }) Context("GetRoleBindingName", func() { It("should return the rolebinding name for the Etcd", func() { - Expect(created.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) + Expect(etcd.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) }) }) Context("IsBackupStoreEnabled", func() { Context("when backup is enabled", func() { It("should return true", func() { - Expect(created.IsBackupStoreEnabled()).To(Equal(true)) + Expect(etcd.IsBackupStoreEnabled()).To(Equal(true)) }) }) Context("when backup is not enabled", func() { It("should return false", func() { - created.Spec.Backup = BackupSpec{} - Expect(created.IsBackupStoreEnabled()).To(Equal(false)) + etcd.Spec.Backup = BackupSpec{} + Expect(etcd.IsBackupStoreEnabled()).To(Equal(false)) }) }) }) @@ -158,13 +132,13 @@ var _ = Describe("Etcd", func() { Context("IsMarkedForDeletion", func() { Context("when deletion timestamp is not set", func() { It("should return false", func() { - Expect(created.IsMarkedForDeletion()).To(Equal(false)) + Expect(etcd.IsMarkedForDeletion()).To(Equal(false)) }) }) Context("when deletion timestamp is set", func() { It("should return true", func() { - created.DeletionTimestamp = &metav1.Time{Time: time.Now()} - Expect(created.IsMarkedForDeletion()).To(Equal(true)) + etcd.DeletionTimestamp = &metav1.Time{Time: time.Now()} + Expect(etcd.IsMarkedForDeletion()).To(Equal(true)) }) }) }) @@ -174,33 +148,33 @@ var _ = Describe("Etcd", func() { suspendEtcdSpecReconcileAnnotation := SuspendEtcdSpecReconcileAnnotation Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { It("should return druid.gardener.cloud/ignore-reconciliation", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ IgnoreReconciliationAnnotation: "true", } - Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&ignoreReconciliationAnnotation)) + Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&ignoreReconciliationAnnotation)) }) }) Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { It("should return druid.gardener.cloud/suspend-etcd-spec-reconcile", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", } - Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) + Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) }) }) Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { It("should return druid.gardener.cloud/suspend-etcd-spec-reconcile", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", IgnoreReconciliationAnnotation: "true", } - Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) + Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) }) }) Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { It("should return nil string pointer", func() { var nilString *string - Expect(created.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(nilString)) + Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(nilString)) }) }) }) @@ -208,32 +182,32 @@ var _ = Describe("Etcd", func() { Context("IsReconciliationSuspended", func() { Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { It("should return true", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", } - Expect(created.IsReconciliationSuspended()).To(Equal(true)) + Expect(etcd.IsReconciliationSuspended()).To(Equal(true)) }) }) Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { It("should return true", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ IgnoreReconciliationAnnotation: "true", } - Expect(created.IsReconciliationSuspended()).To(Equal(true)) + Expect(etcd.IsReconciliationSuspended()).To(Equal(true)) }) }) Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { It("should return true", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", IgnoreReconciliationAnnotation: "true", } - Expect(created.IsReconciliationSuspended()).To(Equal(true)) + Expect(etcd.IsReconciliationSuspended()).To(Equal(true)) }) }) Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { It("should return false", func() { - Expect(created.IsReconciliationSuspended()).To(Equal(false)) + Expect(etcd.IsReconciliationSuspended()).To(Equal(false)) }) }) }) @@ -241,20 +215,20 @@ var _ = Describe("Etcd", func() { Context("AreManagedResourcesProtected", func() { Context("when etcd has annotation druid.gardener.cloud/resource-protection: false", func() { It("should return false", func() { - created.Annotations = map[string]string{ + etcd.Annotations = map[string]string{ ResourceProtectionAnnotation: "false", } - Expect(created.AreManagedResourcesProtected()).To(Equal(false)) + Expect(etcd.AreManagedResourcesProtected()).To(Equal(false)) }) }) Context("when etcd has annotation druid.gardener.cloud/resource-protection: true", func() { It("should return true", func() { - Expect(created.AreManagedResourcesProtected()).To(Equal(true)) + Expect(etcd.AreManagedResourcesProtected()).To(Equal(true)) }) }) Context("when etcd does not have annotation druid.gardener.cloud/resource-protection set", func() { It("should return true", func() { - Expect(created.AreManagedResourcesProtected()).To(Equal(true)) + Expect(etcd.AreManagedResourcesProtected()).To(Equal(true)) }) }) }) @@ -262,31 +236,31 @@ var _ = Describe("Etcd", func() { Context("IsBeingProcessed", func() { Context("when etcd status has lastOperation and its state is Processing", func() { It("should return true", func() { - created.Status.LastOperation = &LastOperation{ + etcd.Status.LastOperation = &LastOperation{ State: LastOperationStateProcessing, } - Expect(created.IsBeingProcessed()).To(Equal(true)) + Expect(etcd.IsBeingProcessed()).To(Equal(true)) }) }) Context("when etcd status has lastOperation and its state is Error", func() { It("should return true", func() { - created.Status.LastOperation = &LastOperation{ + etcd.Status.LastOperation = &LastOperation{ State: LastOperationStateError, } - Expect(created.IsBeingProcessed()).To(Equal(true)) + Expect(etcd.IsBeingProcessed()).To(Equal(true)) }) }) Context("when etcd status has lastOperation and its state is neither Processing or Error", func() { It("should return false", func() { - created.Status.LastOperation = &LastOperation{ + etcd.Status.LastOperation = &LastOperation{ State: LastOperationStateSucceeded, } - Expect(created.IsBeingProcessed()).To(Equal(false)) + Expect(etcd.IsBeingProcessed()).To(Equal(false)) }) }) Context("when etcd status does not have lastOperation populated", func() { It("should return false", func() { - Expect(created.IsBeingProcessed()).To(Equal(false)) + Expect(etcd.IsBeingProcessed()).To(Equal(false)) }) }) }) From 693902e07a0acb60fd7f1e80cb36dc6d82f56a1d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 15 Feb 2024 11:38:41 +0530 Subject: [PATCH 090/235] added IT tests for etcd reconciler --- internal/controller/etcd/reconciler.go | 23 +- internal/controller/etcd/register.go | 33 +- internal/controller/etcd/register_test.go | 146 +++++++++ internal/controller/utils/reconciler.go | 8 + .../mock/controller-runtime/manager/doc.go | 2 + .../mock/controller-runtime/manager/mocks.go | 299 +++++++++++++++++ test/it/controller/etcd/assertions.go | 128 +++++++- test/it/controller/etcd/reconciler_test.go | 306 +++++++++++++++--- test/it/setup/setup.go | 5 + 9 files changed, 873 insertions(+), 77 deletions(-) create mode 100644 internal/controller/etcd/register_test.go create mode 100644 internal/mock/controller-runtime/manager/doc.go create mode 100644 internal/mock/controller-runtime/manager/mocks.go diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index ab808489e..70ea552be 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -103,18 +103,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if result := r.getLatestEtcd(ctx, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result.ReconcileResult() } - reconcileFns := []reconcileFn{ - r.reconcileEtcdDeletion, - r.reconcileSpec, - r.reconcileStatus, - } - // Execute the reconcile functions. runID := uuid.New().String() operatorCtx := component.NewOperatorContext(ctx, r.logger, runID) - for _, fn := range reconcileFns { - if result := fn(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { - return result.ReconcileResult() - } + if result := r.reconcileEtcdDeletion(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() + } + var reconcileSpecResult ctrlutils.ReconcileStepResult + if result := r.reconcileSpec(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { + reconcileSpecResult = result + } + + if result := r.reconcileStatus(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { + return result.ReconcileResult() + } + if reconcileSpecResult.HasErrors() { + return reconcileSpecResult.ReconcileResult() } return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult() } diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 35281e747..c4d7effb5 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -26,16 +26,21 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(10*time.Millisecond, r.config.EtcdStatusSyncPeriod), }). For(&druidv1alpha1.Etcd{}). - WithEventFilter( - predicate.Or( - predicate.And(r.forcedReconcile(), noSpecAndStatusUpdated()), - predicate.And(r.reconcilePermitted(), onlySpecUpdated()), - ), - ) + WithEventFilter(r.buildPredicate()) return builder.Complete(r) } +func (r *Reconciler) buildPredicate() predicate.Predicate { + // Since we do etcd status updates in the same reconciliation flow, this will also generate an event which should be ignored. Any changes in the predicates should ensure that this is not violated. + return predicate.Or( + // If reconcile operation annotation is present but there is no change to spec or status then we should allow reconciliation. Consider a case where the spec reconciliation has failed + predicate.And(r.forcedReconcile(), noSpecAndStatusUpdated()), + // If the reconciliation is allowed and the spec has changed, we should reconcile. + predicate.And(r.reconcilePermitted(), onlySpecUpdated()), + ) +} + func onlySpecUpdated() predicate.Predicate { return predicate.Funcs{ UpdateFunc: func(updateEvent event.UpdateEvent) bool { @@ -57,15 +62,15 @@ func hasSpecChanged(updateEvent event.UpdateEvent) bool { } func hasStatusChanged(updateEvent event.UpdateEvent) bool { - return apiequality.Semantic.DeepEqual(accessEtcdStatus(updateEvent.ObjectNew), accessEtcdStatus(updateEvent.ObjectOld)) -} - -func accessEtcdStatus(object client.Object) *druidv1alpha1.EtcdStatus { - if etcd, ok := object.(*druidv1alpha1.Etcd); !ok { - return nil - } else { - return &etcd.Status + oldEtcd, ok := updateEvent.ObjectOld.(*druidv1alpha1.Etcd) + if !ok { + return false + } + newEtcd, ok := updateEvent.ObjectNew.(*druidv1alpha1.Etcd) + if !ok { + return false } + return !apiequality.Semantic.DeepEqual(oldEtcd.Status, newEtcd.Status) } // reconcilePermitted diff --git a/internal/controller/etcd/register_test.go b/internal/controller/etcd/register_test.go new file mode 100644 index 000000000..b5fa38136 --- /dev/null +++ b/internal/controller/etcd/register_test.go @@ -0,0 +1,146 @@ +package etcd + +import ( + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + testutils "github.com/gardener/etcd-druid/test/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + mockmanager "github.com/gardener/etcd-druid/internal/mock/controller-runtime/manager" +) + +type preCondition struct { + enableEtcdSpecAutoReconcile bool + ignoreOperationAnnotation bool + reconcileAnnotationPresent bool + etcdSpecChanged bool + etcdStatusChanged bool +} + +func TestBuildPredicate(t *testing.T) { + testCases := []struct { + name string + preCondition preCondition + // expected behavior for different event types + shouldAllowCreateEvent bool + shouldAllowDeleteEvent bool + shouldAllowGenericEvent bool + shouldAllowUpdateEvent bool + }{ + // TODO (madhav): remove this test case once ignoreOperationAnnotation has been removed. It has already been marked as deprecated. + { + name: "when ignoreOperationAnnotation is true and only spec has changed", + preCondition: preCondition{ignoreOperationAnnotation: true, reconcileAnnotationPresent: false, etcdSpecChanged: true}, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: true, + shouldAllowUpdateEvent: true, + }, + // TODO (madhav): remove this test case once ignoreOperationAnnotation has been removed. It has already been marked as deprecated. + { + name: "when ignoreOperationAnnotation is true and there is no change to spec but only to status", + preCondition: preCondition{ignoreOperationAnnotation: true, reconcileAnnotationPresent: false, etcdStatusChanged: true}, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: true, + shouldAllowUpdateEvent: false, + }, + { + name: "when enableEtcdSpecAutoReconcile is true and only spec has changed", + preCondition: preCondition{enableEtcdSpecAutoReconcile: true, reconcileAnnotationPresent: false, etcdSpecChanged: true}, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: true, + shouldAllowUpdateEvent: true, + }, + { + name: "when enableEtcdSpecAutoReconcile is true and there is no change to spec but only to status", + preCondition: preCondition{enableEtcdSpecAutoReconcile: true, reconcileAnnotationPresent: false, etcdStatusChanged: true}, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: true, + shouldAllowUpdateEvent: false, + }, + { + name: "when reconcile annotation is present and no change to spec and status", + preCondition: preCondition{reconcileAnnotationPresent: true}, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: true, + shouldAllowUpdateEvent: true, + }, + } + g := NewWithT(t) + etcd := createEtcd() + etcd.Status.Replicas = 1 + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + r := createReconciler(t, tc.preCondition.enableEtcdSpecAutoReconcile, tc.preCondition.ignoreOperationAnnotation) + predicate := r.buildPredicate() + g.Expect(predicate.Create(event.CreateEvent{Object: etcd})).To(Equal(tc.shouldAllowCreateEvent)) + g.Expect(predicate.Delete(event.DeleteEvent{Object: etcd})).To(Equal(tc.shouldAllowDeleteEvent)) + g.Expect(predicate.Generic(event.GenericEvent{Object: etcd})).To(Equal(tc.shouldAllowGenericEvent)) + updatedEtcd := updateEtcd(tc.preCondition, etcd) + g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: etcd, ObjectNew: updatedEtcd})).To(Equal(tc.shouldAllowUpdateEvent)) + }) + } +} + +func createEtcd() *druidv1alpha1.Etcd { + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(3).Build() + etcd.Status = druidv1alpha1.EtcdStatus{ + ObservedGeneration: pointer.Int64(0), + Etcd: &druidv1alpha1.CrossVersionObjectReference{ + Kind: "StatefulSet", + Name: testutils.TestEtcdName, + APIVersion: "apps/v1", + }, + CurrentReplicas: 3, + Replicas: 3, + ReadyReplicas: 3, + Ready: pointer.Bool(true), + } + return etcd +} + +func updateEtcd(preCondition preCondition, originalEtcd *druidv1alpha1.Etcd) *druidv1alpha1.Etcd { + newEtcd := originalEtcd.DeepCopy() + annotations := make(map[string]string) + if preCondition.reconcileAnnotationPresent { + annotations[v1beta1constants.GardenerOperation] = v1beta1constants.GardenerOperationReconcile + newEtcd.SetAnnotations(annotations) + } + if preCondition.etcdSpecChanged { + // made a single change to the spec + newEtcd.Spec.Backup.Image = pointer.String("eu.gcr.io/gardener-project/gardener/etcdbrctl-distroless:v1.0.0") + newEtcd.Generation++ + } + if preCondition.etcdStatusChanged { + // made a single change to the status + newEtcd.Status.ReadyReplicas = 2 + newEtcd.Status.Ready = pointer.Bool(false) + } + return newEtcd +} + +func createReconciler(t *testing.T, enableEtcdSpecAutoReconcile, ignoreOperationAnnotation bool) *Reconciler { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + mgr := mockmanager.NewMockManager(mockCtrl) + fakeClient := fake.NewClientBuilder().Build() + mgr.EXPECT().GetClient().AnyTimes().Return(testutils.NewTestClientBuilder().WithClient(fakeClient).Build()) + mgr.EXPECT().GetEventRecorderFor(gomock.Any()).AnyTimes().Return(nil) + etcdConfig := Config{ + IgnoreOperationAnnotation: ignoreOperationAnnotation, + EnableEtcdSpecAutoReconcile: enableEtcdSpecAutoReconcile, + } + r, err := NewReconcilerWithImageVector(mgr, &etcdConfig, nil) + g.Expect(err).NotTo(HaveOccurred()) + return r +} diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 7c688936e..059a4989b 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -88,6 +88,14 @@ func (r ReconcileStepResult) GetErrors() []error { return r.errs } +func (r ReconcileStepResult) GetResult() ctrl.Result { + return r.result +} + +func (r ReconcileStepResult) HasErrors() bool { + return len(r.errs) > 0 +} + func (r ReconcileStepResult) GetDescription() string { if len(r.errs) > 0 { return fmt.Sprintf("%s %s", r.description, errors.Join(r.errs...).Error()) diff --git a/internal/mock/controller-runtime/manager/doc.go b/internal/mock/controller-runtime/manager/doc.go new file mode 100644 index 000000000..d34952302 --- /dev/null +++ b/internal/mock/controller-runtime/manager/doc.go @@ -0,0 +1,2 @@ +//go:generate mockgen -package manager -destination=mocks.go sigs.k8s.io/controller-runtime/pkg/manager Manager +package manager diff --git a/internal/mock/controller-runtime/manager/mocks.go b/internal/mock/controller-runtime/manager/mocks.go new file mode 100644 index 000000000..dc8dd8d88 --- /dev/null +++ b/internal/mock/controller-runtime/manager/mocks.go @@ -0,0 +1,299 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: sigs.k8s.io/controller-runtime/pkg/manager (interfaces: Manager) + +// Package manager is a generated GoMock package. +package manager + +import ( + context "context" + http "net/http" + reflect "reflect" + + logr "github.com/go-logr/logr" + gomock "github.com/golang/mock/gomock" + meta "k8s.io/apimachinery/pkg/api/meta" + runtime "k8s.io/apimachinery/pkg/runtime" + rest "k8s.io/client-go/rest" + record "k8s.io/client-go/tools/record" + cache "sigs.k8s.io/controller-runtime/pkg/cache" + client "sigs.k8s.io/controller-runtime/pkg/client" + v1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + healthz "sigs.k8s.io/controller-runtime/pkg/healthz" + manager "sigs.k8s.io/controller-runtime/pkg/manager" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// MockManager is a mock of Manager interface. +type MockManager struct { + ctrl *gomock.Controller + recorder *MockManagerMockRecorder +} + +// MockManagerMockRecorder is the mock recorder for MockManager. +type MockManagerMockRecorder struct { + mock *MockManager +} + +// NewMockManager creates a new mock instance. +func NewMockManager(ctrl *gomock.Controller) *MockManager { + mock := &MockManager{ctrl: ctrl} + mock.recorder = &MockManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockManager) EXPECT() *MockManagerMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockManager) Add(arg0 manager.Runnable) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Add indicates an expected call of Add. +func (mr *MockManagerMockRecorder) Add(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockManager)(nil).Add), arg0) +} + +// AddHealthzCheck mocks base method. +func (m *MockManager) AddHealthzCheck(arg0 string, arg1 healthz.Checker) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddHealthzCheck", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddHealthzCheck indicates an expected call of AddHealthzCheck. +func (mr *MockManagerMockRecorder) AddHealthzCheck(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddHealthzCheck", reflect.TypeOf((*MockManager)(nil).AddHealthzCheck), arg0, arg1) +} + +// AddMetricsExtraHandler mocks base method. +func (m *MockManager) AddMetricsExtraHandler(arg0 string, arg1 http.Handler) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddMetricsExtraHandler", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddMetricsExtraHandler indicates an expected call of AddMetricsExtraHandler. +func (mr *MockManagerMockRecorder) AddMetricsExtraHandler(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddMetricsExtraHandler", reflect.TypeOf((*MockManager)(nil).AddMetricsExtraHandler), arg0, arg1) +} + +// AddReadyzCheck mocks base method. +func (m *MockManager) AddReadyzCheck(arg0 string, arg1 healthz.Checker) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddReadyzCheck", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddReadyzCheck indicates an expected call of AddReadyzCheck. +func (mr *MockManagerMockRecorder) AddReadyzCheck(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddReadyzCheck", reflect.TypeOf((*MockManager)(nil).AddReadyzCheck), arg0, arg1) +} + +// Elected mocks base method. +func (m *MockManager) Elected() <-chan struct{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Elected") + ret0, _ := ret[0].(<-chan struct{}) + return ret0 +} + +// Elected indicates an expected call of Elected. +func (mr *MockManagerMockRecorder) Elected() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Elected", reflect.TypeOf((*MockManager)(nil).Elected)) +} + +// GetAPIReader mocks base method. +func (m *MockManager) GetAPIReader() client.Reader { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAPIReader") + ret0, _ := ret[0].(client.Reader) + return ret0 +} + +// GetAPIReader indicates an expected call of GetAPIReader. +func (mr *MockManagerMockRecorder) GetAPIReader() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAPIReader", reflect.TypeOf((*MockManager)(nil).GetAPIReader)) +} + +// GetCache mocks base method. +func (m *MockManager) GetCache() cache.Cache { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCache") + ret0, _ := ret[0].(cache.Cache) + return ret0 +} + +// GetCache indicates an expected call of GetCache. +func (mr *MockManagerMockRecorder) GetCache() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCache", reflect.TypeOf((*MockManager)(nil).GetCache)) +} + +// GetClient mocks base method. +func (m *MockManager) GetClient() client.Client { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClient") + ret0, _ := ret[0].(client.Client) + return ret0 +} + +// GetClient indicates an expected call of GetClient. +func (mr *MockManagerMockRecorder) GetClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockManager)(nil).GetClient)) +} + +// GetConfig mocks base method. +func (m *MockManager) GetConfig() *rest.Config { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfig") + ret0, _ := ret[0].(*rest.Config) + return ret0 +} + +// GetConfig indicates an expected call of GetConfig. +func (mr *MockManagerMockRecorder) GetConfig() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfig", reflect.TypeOf((*MockManager)(nil).GetConfig)) +} + +// GetControllerOptions mocks base method. +func (m *MockManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetControllerOptions") + ret0, _ := ret[0].(v1alpha1.ControllerConfigurationSpec) + return ret0 +} + +// GetControllerOptions indicates an expected call of GetControllerOptions. +func (mr *MockManagerMockRecorder) GetControllerOptions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetControllerOptions", reflect.TypeOf((*MockManager)(nil).GetControllerOptions)) +} + +// GetEventRecorderFor mocks base method. +func (m *MockManager) GetEventRecorderFor(arg0 string) record.EventRecorder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEventRecorderFor", arg0) + ret0, _ := ret[0].(record.EventRecorder) + return ret0 +} + +// GetEventRecorderFor indicates an expected call of GetEventRecorderFor. +func (mr *MockManagerMockRecorder) GetEventRecorderFor(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEventRecorderFor", reflect.TypeOf((*MockManager)(nil).GetEventRecorderFor), arg0) +} + +// GetFieldIndexer mocks base method. +func (m *MockManager) GetFieldIndexer() client.FieldIndexer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFieldIndexer") + ret0, _ := ret[0].(client.FieldIndexer) + return ret0 +} + +// GetFieldIndexer indicates an expected call of GetFieldIndexer. +func (mr *MockManagerMockRecorder) GetFieldIndexer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFieldIndexer", reflect.TypeOf((*MockManager)(nil).GetFieldIndexer)) +} + +// GetLogger mocks base method. +func (m *MockManager) GetLogger() logr.Logger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLogger") + ret0, _ := ret[0].(logr.Logger) + return ret0 +} + +// GetLogger indicates an expected call of GetLogger. +func (mr *MockManagerMockRecorder) GetLogger() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogger", reflect.TypeOf((*MockManager)(nil).GetLogger)) +} + +// GetRESTMapper mocks base method. +func (m *MockManager) GetRESTMapper() meta.RESTMapper { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRESTMapper") + ret0, _ := ret[0].(meta.RESTMapper) + return ret0 +} + +// GetRESTMapper indicates an expected call of GetRESTMapper. +func (mr *MockManagerMockRecorder) GetRESTMapper() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRESTMapper", reflect.TypeOf((*MockManager)(nil).GetRESTMapper)) +} + +// GetScheme mocks base method. +func (m *MockManager) GetScheme() *runtime.Scheme { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScheme") + ret0, _ := ret[0].(*runtime.Scheme) + return ret0 +} + +// GetScheme indicates an expected call of GetScheme. +func (mr *MockManagerMockRecorder) GetScheme() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScheme", reflect.TypeOf((*MockManager)(nil).GetScheme)) +} + +// GetWebhookServer mocks base method. +func (m *MockManager) GetWebhookServer() *webhook.Server { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetWebhookServer") + ret0, _ := ret[0].(*webhook.Server) + return ret0 +} + +// GetWebhookServer indicates an expected call of GetWebhookServer. +func (mr *MockManagerMockRecorder) GetWebhookServer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWebhookServer", reflect.TypeOf((*MockManager)(nil).GetWebhookServer)) +} + +// SetFields mocks base method. +func (m *MockManager) SetFields(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetFields", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetFields indicates an expected call of SetFields. +func (mr *MockManagerMockRecorder) SetFields(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFields", reflect.TypeOf((*MockManager)(nil).SetFields), arg0) +} + +// Start mocks base method. +func (m *MockManager) Start(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *MockManagerMockRecorder) Start(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockManager)(nil).Start), arg0) +} diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index a7c27c73f..b8cbe1e1c 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -2,6 +2,7 @@ package etcd import ( "context" + "errors" "fmt" "slices" "sync" @@ -17,6 +18,7 @@ import ( "github.com/gardener/etcd-druid/test/it/setup" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -217,7 +219,7 @@ func assertETCDObservedGeneration(t *testing.T, cl client.Client, etcdObjectKey t.Logf("observedGeneration correctly set to %s", logPointerTypeToString[int64](expectedObservedGeneration)) } -func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedLastOperation *druidv1alpha1.LastOperation, expectedLastErrors []druidv1alpha1.LastError, timeout, pollInterval time.Duration) { +func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedLastOperation *druidv1alpha1.LastOperation, expectedLastErrorCodes []string, timeout, pollInterval time.Duration) { g := NewWithT(t) checkFn := func() error { etcdInstance := &druidv1alpha1.Etcd{} @@ -238,13 +240,12 @@ func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl cl } // For comparing last errors, it is sufficient to compare their length and their error codes. - expectedErrorCodes := getErrorCodesFromLastErrors(expectedLastErrors) - slices.Sort(expectedErrorCodes) + slices.Sort(expectedLastErrorCodes) actualErrorCodes := getErrorCodesFromLastErrors(etcdInstance.Status.LastErrors) slices.Sort(actualErrorCodes) - if !slices.Equal(expectedErrorCodes, actualErrorCodes) { - return fmt.Errorf("expected lastErrors to be %v, found %v", expectedLastErrors, etcdInstance.Status.LastErrors) + if !slices.Equal(expectedLastErrorCodes, actualErrorCodes) { + return fmt.Errorf("expected lastErrors to be %v, found %v", expectedLastErrorCodes, etcdInstance.Status.LastErrors) } return nil } @@ -311,10 +312,91 @@ func assertETCDFinalizer(t *testing.T, cl client.Client, etcdObjectKey client.Ob t.Log(msg) } -func getErrorCodesFromLastErrors(lastErrors []druidv1alpha1.LastError) []druidv1alpha1.ErrorCode { - errorCodes := make([]druidv1alpha1.ErrorCode, 0, len(lastErrors)) +func assertETCDMemberStatuses(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedMembers []druidv1alpha1.EtcdMemberStatus, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + etcdInstance := &druidv1alpha1.Etcd{} + if err := cl.Get(context.Background(), etcdObjectKey, etcdInstance); err != nil { + return err + } + actualMembers := etcdInstance.Status.Members + if len(actualMembers) != len(expectedMembers) { + return fmt.Errorf("expected %d members, found %d", len(expectedMembers), len(actualMembers)) + } + var memberErr error + for _, expectedMember := range expectedMembers { + foundMember, err := findMatchingMemberStatus(actualMembers, expectedMember) + if err != nil { + memberErr = errors.Join(memberErr, err) + } else { + t.Logf("member with [id:%s, name:%s, role:%s, status:%s] matches expected member", logPointerTypeToString(foundMember.ID), foundMember.Name, logPointerTypeToString(foundMember.Role), foundMember.Status) + } + } + return memberErr + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + t.Logf("asserted that etcd member statuses matches expected members: %v", expectedMembers) +} + +func assertETCDStatusConditions(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedConditions []druidv1alpha1.Condition, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + etcdInstance := &druidv1alpha1.Etcd{} + if err := cl.Get(context.Background(), etcdObjectKey, etcdInstance); err != nil { + return err + } + actualConditions := etcdInstance.Status.Conditions + var condErr error + for _, expectedCondition := range expectedConditions { + foundCondition, err := findMatchingCondition(actualConditions, expectedCondition) + if err != nil { + condErr = errors.Join(condErr, err) + } else { + t.Logf("found condition with [type:%s, status:%s] matches expected condition", foundCondition.Type, foundCondition.Status) + } + } + if condErr != nil { + t.Logf("not all conditions matched: %v. will retry matching all expected conditions", condErr) + } + return condErr + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + t.Logf("asserted that etcd status conditions matches expected conditions: %v", expectedConditions) +} + +func assertETCDStatusFieldsDerivedFromStatefulSet(ctx context.Context, t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, stsObjectKey client.ObjectKey, timeout, pollInterval time.Duration, expectedReadyStatus bool) { + g := NewWithT(t) + checkFn := func() error { + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, stsObjectKey, sts); err != nil { + return err + } + etcdInstance := &druidv1alpha1.Etcd{} + if err := cl.Get(context.Background(), etcdObjectKey, etcdInstance); err != nil { + return err + } + if etcdInstance.Status.ReadyReplicas != sts.Status.ReadyReplicas { + return fmt.Errorf("expected readyReplicas to be %d, found %d", sts.Status.ReadyReplicas, etcdInstance.Status.ReadyReplicas) + } + if etcdInstance.Status.CurrentReplicas != sts.Status.CurrentReplicas { + return fmt.Errorf("expected currentReplicas to be %d, found %d", sts.Status.CurrentReplicas, etcdInstance.Status.CurrentReplicas) + } + if etcdInstance.Status.Replicas != *sts.Spec.Replicas { + return fmt.Errorf("expected replicas to be %d, found %d", sts.Spec.Replicas, etcdInstance.Status.Replicas) + } + if utils.TypeDeref[bool](etcdInstance.Status.Ready, false) != expectedReadyStatus { + return fmt.Errorf("expected ready to be %t, found %s", expectedReadyStatus, logPointerTypeToString[bool](etcdInstance.Status.Ready)) + } + return nil + } + g.Eventually(checkFn).WithContext(ctx).Within(timeout).WithPolling(pollInterval).Should(BeNil()) + t.Logf("asserted that etcd status fields are correctly derived from statefulset: %s", stsObjectKey.Name) +} + +func getErrorCodesFromLastErrors(lastErrors []druidv1alpha1.LastError) []string { + errorCodes := make([]string, 0, len(lastErrors)) for _, lastErr := range lastErrors { - errorCodes = append(errorCodes, lastErr.Code) + errorCodes = append(errorCodes, string(lastErr.Code)) } return errorCodes } @@ -325,3 +407,33 @@ func logPointerTypeToString[T any](val *T) string { } return fmt.Sprintf("%v", *val) } + +func findMatchingCondition(actualConditions []druidv1alpha1.Condition, conditionToMatch druidv1alpha1.Condition) (*druidv1alpha1.Condition, error) { + for _, actualCondition := range actualConditions { + if actualCondition.Type == conditionToMatch.Type { + if actualCondition.Status != conditionToMatch.Status { + return nil, fmt.Errorf("for condition type: %s, expected status to be %s, found %s", conditionToMatch.Type, conditionToMatch.Status, actualCondition.Status) + } + return &actualCondition, nil + } + } + return nil, fmt.Errorf("condition type: %s not found", conditionToMatch.Type) +} + +func findMatchingMemberStatus(actualMembers []druidv1alpha1.EtcdMemberStatus, memberStatusToMatch druidv1alpha1.EtcdMemberStatus) (*druidv1alpha1.EtcdMemberStatus, error) { + for _, actualMember := range actualMembers { + if *actualMember.ID == *memberStatusToMatch.ID { + if actualMember.Name != memberStatusToMatch.Name { + return nil, fmt.Errorf("for member with id: %s, expected name to be %s, found %s", logPointerTypeToString(actualMember.ID), memberStatusToMatch.Name, actualMember.Name) + } + if *actualMember.Role != *memberStatusToMatch.Role { + return nil, fmt.Errorf("for member with id: %s, expected role to be %s, found %s", logPointerTypeToString(actualMember.ID), logPointerTypeToString(memberStatusToMatch.Role), logPointerTypeToString(actualMember.Role)) + } + if actualMember.Status != memberStatusToMatch.Status { + return nil, fmt.Errorf("for member with id: %s, expected status to be %s, found %s", logPointerTypeToString(actualMember.ID), memberStatusToMatch.Status, actualMember.Status) + } + return &actualMember, nil + } + } + return nil, fmt.Errorf("member with id: %s not found", logPointerTypeToString(memberStatusToMatch.ID)) +} diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index e6c2c0a08..4e3999981 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -19,30 +19,45 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/featuregate" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + + testclock "k8s.io/utils/clock/testing" ) const testNamespacePrefix = "etcd-reconciler-test-" -/* - status update check: - * create etcd resource and let sts be created - * assert the etcd status - * update sts status and then assert etcd status again - - deletion flow -done - spec reconcile flow -done -*/ - -// ------------------------------ reconcile spec tests ------------------------------ -func TestEtcdReconcilerSpecWithoutAutoReconcile(t *testing.T) { +// TestSuiteWithDefaultIntegrationTestEnvironment runs the etcd reconciler tests with the default integration test environment. +// All tests defined in this suite share the same integration test environment. +func TestSuiteWithDefaultIntegrationTestEnvironment(t *testing.T) { itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) defer itTestEnvCloser(t) reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) + tests := []struct { + name string + testFn func(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) + }{ + {"test spec reconciliation without auto-reconcile", testEtcdReconcileSpecWithoutAutoReconcile}, + {"test deletion of all etcd resources when etcd marked for deletion", testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion}, + {"test etcd status", testEtcdStatus}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.testFn(t, reconcilerTestEnv) + }) + } +} + +// ------------------------------ reconcile spec tests ------------------------------ +func testEtcdReconcileSpecWithoutAutoReconcile(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { tests := []struct { name string fn func(t *testing.T, testNamespace string, reconcilerTestEnv ReconcilerTestEnv) @@ -52,8 +67,6 @@ func TestEtcdReconcilerSpecWithoutAutoReconcile(t *testing.T) { {"should not reconcile spec when reconciliation is suspended", testWhenReconciliationIsSuspended}, {"should not reconcile when no reconcile operation annotation is set", testWhenNoReconcileOperationAnnotationIsSet}, } - - t.Parallel() for _, test := range tests { t.Run(test.name, func(t *testing.T) { testNs := createTestNamespaceName(t) @@ -139,12 +152,7 @@ func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTest Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateError, } - expectedLastErrs := []druidv1alpha1.LastError{ - { - Code: "ERR_SYNC_CLIENT_SERVICE", - }, - } - assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, expectedLastErrs, 5*time.Second, 1*time.Second) + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, []string{"ERR_SYNC_CLIENT_SERVICE"}, 5*time.Second, 1*time.Second) } func testWhenReconciliationIsSuspended(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { @@ -189,15 +197,10 @@ func testWhenNoReconcileOperationAnnotationIsSet(t *testing.T, testNs string, re } // ------------------------------ reconcile deletion tests ------------------------------ -func TestDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T) { - // ***************** setup ***************** - itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) - defer itTestEnvCloser(t) - reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) - +func testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { // --------------------------- create test namespace --------------------------- testNs := createTestNamespaceName(t) - itTestEnv.CreateTestNamespace(testNs) + reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs) t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) // ---------------------------- create etcd instance -------------------------- @@ -211,7 +214,7 @@ func TestDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T) { }).Build() ctx := context.Background() cl := reconcilerTestEnv.itTestEnv.GetClient() - createEtcdInstanceWithFinalizer(ctx, t, reconcilerTestEnv, etcdInstance) + createAndAssertEtcdAndAllManagedResources(ctx, t, reconcilerTestEnv, etcdInstance) // ***************** test etcd deletion flow ***************** // mark etcd for deletion @@ -261,7 +264,7 @@ func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi g.Expect(testutils.CreateSecrets(ctx, cl, testNs, etcdInstance.Spec.Backup.Store.SecretRef.Name)).To(Succeed()) t.Logf("successfully created backup secrets for etcd instance: %s", etcdInstance.Name) - createEtcdInstanceWithFinalizer(ctx, t, reconcilerTestEnv, etcdInstance) + createAndAssertEtcdAndAllManagedResources(ctx, t, reconcilerTestEnv, etcdInstance) // ******************************* test etcd deletion flow ******************************* // mark etcd for deletion @@ -283,29 +286,171 @@ func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi Type: druidv1alpha1.LastOperationTypeDelete, State: druidv1alpha1.LastOperationStateError, } - expectedLastErrors := []druidv1alpha1.LastError{ - { - Code: "ERR_DELETE_CLIENT_SERVICE", - }, - { - Code: "ERR_DELETE_SNAPSHOT_LEASE", - }, - } - assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, expectedLastErrors, 2*time.Minute, 2*time.Second) + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, []string{"ERR_DELETE_CLIENT_SERVICE", "ERR_DELETE_SNAPSHOT_LEASE"}, 2*time.Minute, 2*time.Second) // assert that the finalizer has not been removed as all resources have not been deleted yet. assertETCDFinalizer(t, cl, client.ObjectKeyFromObject(etcdInstance), true, 10*time.Second, 2*time.Second) } // ------------------------------ reconcile status tests ------------------------------ +func testEtcdStatus(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { + tests := []struct { + name string + fn func(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) + }{ + {"check assertions when all member leases are active", testConditionsAndMembersWhenAllMemberLeasesAreActive}, + {"assert that data volume condition reflects pvc error event", testEtcdStatusReflectsPVCErrorEvent}, + {"test when all sts replicas are ready", testEtcdStatusIsInSyncWithStatefulSetStatusWhenAllReplicasAreReady}, + {"test when not all sts replicas are ready", testEtcdStatusIsInSyncWithStatefulSetStatusWhenNotAllReplicasAreReady}, + {"test when sts current revision is older than update revision", testEtcdStatusIsInSyncWithStatefulSetStatusWhenCurrentRevisionIsOlderThanUpdateRevision}, + /* + Additional tests to check the conditions and member status should be added when we solve https://github.com/gardener/etcd-druid/issues/645 + Currently only one happy-state test has been added as a template for other tests to follow once the conditions are refactored. + Writing additional tests for member status and conditions would be a waste of time as they will have to be modified again. + */ + } + + ctx := context.Background() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // --------------------------- create test namespace --------------------------- + testNs := createTestNamespaceName(t) + reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs) + t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) + // ---------------------------- create etcd instance -------------------------- + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + WithAnnotations(map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + }).Build() + createAndAssertEtcdAndAllManagedResources(ctx, t, reconcilerTestEnv, etcdInstance) + test.fn(t, etcdInstance, reconcilerTestEnv) + }) + } +} + +func testConditionsAndMembersWhenAllMemberLeasesAreActive(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { + memberLeaseNames := etcd.GetMemberLeaseNames() + testNs := etcd.Namespace + clock := testclock.NewFakeClock(time.Now().Round(time.Second)) + mlcs := []etcdMemberLeaseConfig{ + {name: memberLeaseNames[0], memberID: generateRandomAlphanumericString(t, 8), role: druidv1alpha1.EtcdRoleMember, renewTime: &metav1.MicroTime{Time: clock.Now().Add(-time.Second * 30)}}, + {name: memberLeaseNames[1], memberID: generateRandomAlphanumericString(t, 8), role: druidv1alpha1.EtcdRoleLeader, renewTime: &metav1.MicroTime{Time: clock.Now()}}, + {name: memberLeaseNames[2], memberID: generateRandomAlphanumericString(t, 8), role: druidv1alpha1.EtcdRoleMember, renewTime: &metav1.MicroTime{Time: clock.Now().Add(-time.Second * 30)}}, + } + updateMemberLeaseSpec(context.Background(), t, reconcilerTestEnv.itTestEnv.GetClient(), testNs, mlcs) + // ******************************* test etcd status update flow ******************************* + expectedConditions := []druidv1alpha1.Condition{ + {Type: druidv1alpha1.ConditionTypeReady, Status: druidv1alpha1.ConditionTrue}, + {Type: druidv1alpha1.ConditionTypeAllMembersReady, Status: druidv1alpha1.ConditionTrue}, + {Type: druidv1alpha1.ConditionTypeDataVolumesReady, Status: druidv1alpha1.ConditionTrue}, + } + cl := reconcilerTestEnv.itTestEnv.GetClient() + assertETCDStatusConditions(t, cl, client.ObjectKeyFromObject(etcd), expectedConditions, 2*time.Minute, 2*time.Second) + expectedMemberStatuses := []druidv1alpha1.EtcdMemberStatus{ + {Name: mlcs[0].name, ID: pointer.String(mlcs[0].memberID), Role: &mlcs[0].role, Status: druidv1alpha1.EtcdMemberStatusReady}, + {Name: mlcs[1].name, ID: pointer.String(mlcs[1].memberID), Role: &mlcs[1].role, Status: druidv1alpha1.EtcdMemberStatusReady}, + {Name: mlcs[2].name, ID: pointer.String(mlcs[2].memberID), Role: &mlcs[2].role, Status: druidv1alpha1.EtcdMemberStatusReady}, + } + assertETCDMemberStatuses(t, cl, client.ObjectKeyFromObject(etcd), expectedMemberStatuses, 2*time.Minute, 2*time.Second) +} + +func testEtcdStatusReflectsPVCErrorEvent(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { + g := NewWithT(t) + testNs := etcd.Namespace + cl := reconcilerTestEnv.itTestEnv.GetClient() + ctx := context.Background() + sts := &appsv1.StatefulSet{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: etcd.Name, Namespace: testNs}, sts)).To(Succeed()) + g.Expect(sts.Spec.VolumeClaimTemplates).To(HaveLen(1)) + // create the pvcs + pvcs := createPVCs(ctx, t, cl, sts) + // create the pvc warning event for one of the pvc + targetPvc := pvcs[0] + targetPVName := fmt.Sprintf("pv-%s", generateRandomAlphanumericString(t, 16)) + const eventReason = "FailedAttachVolume" + eventMessage := fmt.Sprintf("Multi-Attach error for volume %s. Volume is already exclusively attached to one node and can't be attached to another", targetPVName) + createPVCWarningEvent(ctx, t, cl, testNs, targetPvc.Name, eventReason, eventMessage) + // ******************************* test etcd status update flow ******************************* + expectedConditions := []druidv1alpha1.Condition{ + {Type: druidv1alpha1.ConditionTypeDataVolumesReady, Status: druidv1alpha1.ConditionFalse, Reason: eventReason, Message: eventMessage}, + } + assertETCDStatusConditions(t, cl, client.ObjectKeyFromObject(etcd), expectedConditions, 5*time.Minute, 2*time.Second) +} + +func testEtcdStatusIsInSyncWithStatefulSetStatusWhenAllReplicasAreReady(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { + g := NewWithT(t) + cl := reconcilerTestEnv.itTestEnv.GetClient() + ctx := reconcilerTestEnv.itTestEnv.GetContext() + sts := &appsv1.StatefulSet{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace}, sts)).To(Succeed()) + stsCopy := sts.DeepCopy() + stsReplicas := *sts.Spec.Replicas + stsCopy.Status.ReadyReplicas = stsReplicas + stsCopy.Status.CurrentReplicas = stsReplicas + stsCopy.Status.Replicas = stsReplicas + stsCopy.Status.ObservedGeneration = stsCopy.Generation + stsCopy.Status.CurrentRevision = fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 2)) + stsCopy.Status.UpdateRevision = stsCopy.Status.CurrentRevision + stsCopy.Status.UpdatedReplicas = stsReplicas + g.Expect(cl.Status().Update(ctx, stsCopy)).To(Succeed()) + // assert etcd status + assertETCDStatusFieldsDerivedFromStatefulSet(ctx, t, cl, client.ObjectKeyFromObject(etcd), client.ObjectKeyFromObject(stsCopy), 2*time.Minute, 2*time.Second, true) +} + +func testEtcdStatusIsInSyncWithStatefulSetStatusWhenNotAllReplicasAreReady(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { + g := NewWithT(t) + cl := reconcilerTestEnv.itTestEnv.GetClient() + ctx := reconcilerTestEnv.itTestEnv.GetContext() + sts := &appsv1.StatefulSet{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace}, sts)).To(Succeed()) + stsCopy := sts.DeepCopy() + stsReplicas := *sts.Spec.Replicas + stsCopy.Status.ReadyReplicas = stsReplicas - 1 + stsCopy.Status.CurrentReplicas = stsReplicas + stsCopy.Status.Replicas = stsReplicas + stsCopy.Status.ObservedGeneration = stsCopy.Generation + stsCopy.Status.CurrentRevision = fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 2)) + stsCopy.Status.UpdateRevision = stsCopy.Status.CurrentRevision + stsCopy.Status.UpdatedReplicas = stsReplicas + g.Expect(cl.Status().Update(ctx, stsCopy)).To(Succeed()) + // assert etcd status + assertETCDStatusFieldsDerivedFromStatefulSet(ctx, t, cl, client.ObjectKeyFromObject(etcd), client.ObjectKeyFromObject(stsCopy), 2*time.Minute, 2*time.Second, false) +} + +func testEtcdStatusIsInSyncWithStatefulSetStatusWhenCurrentRevisionIsOlderThanUpdateRevision(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { + g := NewWithT(t) + cl := reconcilerTestEnv.itTestEnv.GetClient() + ctx := reconcilerTestEnv.itTestEnv.GetContext() + sts := &appsv1.StatefulSet{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace}, sts)).To(Succeed()) + stsCopy := sts.DeepCopy() + stsReplicas := *sts.Spec.Replicas + stsCopy.Status.ReadyReplicas = stsReplicas + stsCopy.Status.CurrentReplicas = stsReplicas + stsCopy.Status.Replicas = stsReplicas + stsCopy.Status.ObservedGeneration = stsCopy.Generation + stsCopy.Status.CurrentRevision = fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 2)) + stsCopy.Status.UpdateRevision = fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 2)) + stsCopy.Status.UpdatedReplicas = stsReplicas - 1 + g.Expect(cl.Status().Update(ctx, stsCopy)).To(Succeed()) + // assert etcd status + assertETCDStatusFieldsDerivedFromStatefulSet(ctx, t, cl, client.ObjectKeyFromObject(etcd), client.ObjectKeyFromObject(stsCopy), 2*time.Minute, 2*time.Second, false) +} // ------------------------- Helper functions ------------------------- func createTestNamespaceName(t *testing.T) string { - b := make([]byte, 4) + namespaceSuffix := generateRandomAlphanumericString(t, 4) + return fmt.Sprintf("%s-%s", testNamespacePrefix, namespaceSuffix) +} + +func generateRandomAlphanumericString(t *testing.T, length int) string { + b := make([]byte, length) _, err := rand.Read(b) g := NewWithT(t) g.Expect(err).ToNot(HaveOccurred()) - namespaceSuffix := hex.EncodeToString(b) - return fmt.Sprintf("%s-%s", testNamespacePrefix, namespaceSuffix) + return hex.EncodeToString(b) } func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTestEnv, autoReconcile bool) ReconcilerTestEnv { @@ -324,6 +469,10 @@ func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTe FeatureGates: map[featuregate.Feature]bool{ features.UseEtcdWrapper: true, }, + EtcdMember: etcd.EtcdMemberConfig{ + NotReadyThreshold: 5 * time.Minute, + UnknownThreshold: 1 * time.Minute, + }, }, assets.CreateImageVector(g)) g.Expect(err).ToNot(HaveOccurred()) g.Expect(reconciler.RegisterWithManager(mgr)).To(Succeed()) @@ -336,18 +485,18 @@ func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTe } } -func createEtcdInstanceWithFinalizer(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { +func createAndAssertEtcdAndAllManagedResources(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { g := NewWithT(t) cl := reconcilerTestEnv.itTestEnv.GetClient() // create etcdInstance resource g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) - t.Logf("trigggered creation of etcd instance: %s, waiting for resources to be created...", etcdInstance.Name) + t.Logf("trigggered creation of etcd instance: {name: %s, namespace: %s}, waiting for resources to be created...", etcdInstance.Name, etcdInstance.Namespace) // ascertain that all etcd resources are created assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 3*time.Minute, 2*time.Second) - t.Logf("successfully created all resources for etcd instance: %s", etcdInstance.Name) + t.Logf("successfully created all resources for etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) // add finalizer addFinalizer(ctx, g, cl, client.ObjectKeyFromObject(etcdInstance)) - t.Logf("successfully added finalizer to etcd instance: %s", etcdInstance.Name) + t.Logf("successfully added finalizer to etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) } func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey client.ObjectKey) { @@ -355,3 +504,70 @@ func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey g.Expect(cl.Get(ctx, etcdObjectKey, etcdInstance)).To(Succeed()) g.Expect(controllerutils.AddFinalizers(ctx, cl, etcdInstance, common.FinalizerName)).To(Succeed()) } + +type etcdMemberLeaseConfig struct { + name string + memberID string + role druidv1alpha1.EtcdRole + renewTime *metav1.MicroTime +} + +func updateMemberLeaseSpec(ctx context.Context, t *testing.T, cl client.Client, namespace string, memberLeaseConfigs []etcdMemberLeaseConfig) { + g := NewWithT(t) + for _, config := range memberLeaseConfigs { + lease := &coordinationv1.Lease{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: config.name, Namespace: namespace}, lease)).To(Succeed()) + updatedLease := lease.DeepCopy() + updatedLease.Spec.HolderIdentity = pointer.String(fmt.Sprintf("%s:%s", config.memberID, config.role)) + updatedLease.Spec.RenewTime = config.renewTime + g.Expect(cl.Update(ctx, updatedLease)).To(Succeed()) + t.Logf("successfully updated member lease %s with holderIdentity: %s", config.name, *updatedLease.Spec.HolderIdentity) + } +} + +func createPVCs(ctx context.Context, t *testing.T, cl client.Client, sts *appsv1.StatefulSet) []*corev1.PersistentVolumeClaim { + g := NewWithT(t) + pvcs := make([]*corev1.PersistentVolumeClaim, 0, int(*sts.Spec.Replicas)) + volClaimName := sts.Spec.VolumeClaimTemplates[0].Name + for i := 0; i < int(*sts.Spec.Replicas); i++ { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%d", volClaimName, sts.Name, i), + Namespace: sts.Namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + g.Expect(cl.Create(ctx, pvc)).To(Succeed()) + pvcs = append(pvcs, pvc) + t.Logf("successfully created pvc: %s", pvc.Name) + } + return pvcs +} + +func createPVCWarningEvent(ctx context.Context, t *testing.T, cl client.Client, namespace, pvcName, reason, message string) { + g := NewWithT(t) + event := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-event-", pvcName), + Namespace: namespace, + }, + InvolvedObject: corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: pvcName, + Namespace: namespace, + APIVersion: "v1", + }, + Reason: reason, + Message: message, + Type: corev1.EventTypeWarning, + } + g.Expect(cl.Create(ctx, event)).To(Succeed()) + t.Logf("successfully created warning event for pvc: %s", pvcName) +} diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index 9f9f3763f..bd44333bf 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -30,6 +30,7 @@ type IntegrationTestEnv interface { GetClient() client.Client CreateTestNamespace(name string) GetLogger() logr.Logger + GetContext() context.Context } type itTestEnv struct { @@ -94,6 +95,10 @@ func (t *itTestEnv) GetLogger() logr.Logger { return t.logger } +func (t *itTestEnv) GetContext() context.Context { + return t.ctx +} + func (t *itTestEnv) prepareScheme() *k8sruntime.Scheme { t.g.Expect(druidv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) return scheme.Scheme From 1a8da5972f7720a11dbcd09558b0cae7a7214073 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 15 Feb 2024 16:05:18 +0530 Subject: [PATCH 091/235] changed the test name and description --- test/it/controller/etcd/reconciler_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 4e3999981..e61813823 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -35,6 +35,7 @@ import ( const testNamespacePrefix = "etcd-reconciler-test-" // TestSuiteWithDefaultIntegrationTestEnvironment runs the etcd reconciler tests with the default integration test environment. +// No auto-reconcile is set, to reconcile one needs to set the GardenerOperation annotation. // All tests defined in this suite share the same integration test environment. func TestSuiteWithDefaultIntegrationTestEnvironment(t *testing.T) { itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) @@ -45,9 +46,9 @@ func TestSuiteWithDefaultIntegrationTestEnvironment(t *testing.T) { name string testFn func(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) }{ - {"test spec reconciliation without auto-reconcile", testEtcdReconcileSpecWithoutAutoReconcile}, + {"test spec reconciliation without auto-reconcile", testEtcdReconcileSpec}, {"test deletion of all etcd resources when etcd marked for deletion", testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion}, - {"test etcd status", testEtcdStatus}, + {"test etcd status reconciliation", testEtcdStatus}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -57,7 +58,7 @@ func TestSuiteWithDefaultIntegrationTestEnvironment(t *testing.T) { } // ------------------------------ reconcile spec tests ------------------------------ -func testEtcdReconcileSpecWithoutAutoReconcile(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { +func testEtcdReconcileSpec(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { tests := []struct { name string fn func(t *testing.T, testNamespace string, reconcilerTestEnv ReconcilerTestEnv) From c3bee7dddbc32e932a0d3434e7b944c9acfea25e Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 16 Feb 2024 14:00:18 +0530 Subject: [PATCH 092/235] Changes to api/v1alpha1 helper functions --- api/v1alpha1/types_etcd.go | 7 ++++++- api/v1alpha1/types_etcd_test.go | 10 +++++----- internal/webhook/sentinel/handler.go | 12 ++++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index b62e84bc3..a2f5ab7ec 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -589,11 +589,15 @@ func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { return nil } +// IsReconciliationSuspended returns true if the etcd resource has the annotation set to suspend spec reconciliation, +// else returns false. func (e *Etcd) IsReconciliationSuspended() bool { suspendReconcileAnnotKey := e.GetSuspendEtcdSpecReconcileAnnotationKey() return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) } +// AreManagedResourcesProtected returns true if the etcd resource has the resource protection annotation set to true, +// else returns false. func (e *Etcd) AreManagedResourcesProtected() bool { if metav1.HasAnnotation(e.ObjectMeta, ResourceProtectionAnnotation) { return e.GetAnnotations()[ResourceProtectionAnnotation] != "false" @@ -601,7 +605,8 @@ func (e *Etcd) AreManagedResourcesProtected() bool { return true } -func (e *Etcd) IsBeingProcessed() bool { +// IsReconciliationInProgress returns true if the etcd resource is currently being reconciled, else returns false. +func (e *Etcd) IsReconciliationInProgress() bool { return e.Status.LastOperation != nil && (e.Status.LastOperation.State == LastOperationStateProcessing || e.Status.LastOperation.State == LastOperationStateError) diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index ee21234d0..8d1541e05 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -233,13 +233,13 @@ var _ = Describe("Etcd", func() { }) }) - Context("IsBeingProcessed", func() { + Context("IsReconciliationInProgress", func() { Context("when etcd status has lastOperation and its state is Processing", func() { It("should return true", func() { etcd.Status.LastOperation = &LastOperation{ State: LastOperationStateProcessing, } - Expect(etcd.IsBeingProcessed()).To(Equal(true)) + Expect(etcd.IsReconciliationInProgress()).To(Equal(true)) }) }) Context("when etcd status has lastOperation and its state is Error", func() { @@ -247,7 +247,7 @@ var _ = Describe("Etcd", func() { etcd.Status.LastOperation = &LastOperation{ State: LastOperationStateError, } - Expect(etcd.IsBeingProcessed()).To(Equal(true)) + Expect(etcd.IsReconciliationInProgress()).To(Equal(true)) }) }) Context("when etcd status has lastOperation and its state is neither Processing or Error", func() { @@ -255,12 +255,12 @@ var _ = Describe("Etcd", func() { etcd.Status.LastOperation = &LastOperation{ State: LastOperationStateSucceeded, } - Expect(etcd.IsBeingProcessed()).To(Equal(false)) + Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) }) }) Context("when etcd status does not have lastOperation populated", func() { It("should return false", func() { - Expect(etcd.IsBeingProcessed()).To(Equal(false)) + Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) }) }) }) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 99ce47424..1bccf3b56 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -55,7 +55,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R err error ) - log := h.logger.WithValues("resourceKind", req.Kind.Kind, "name", req.Name, "namespace", req.Namespace, "operation", req.Operation, "user", req.UserInfo.Username) + log := h.logger.WithValues("resourceGroup", req.Kind.Group, "resourceKind", req.Kind.Kind, "name", req.Name, "namespace", req.Namespace, "operation", req.Operation, "user", req.UserInfo.Username) log.Info("Sentinel webhook invoked") if req.Operation == admissionv1.Delete { @@ -111,13 +111,13 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) } - // allow operations on resources if any etcd operation is currently under processing, but only by etcd-druid, - // and allow exempt service accounts to make changes to resources, but only if etcd is not currently under processing. - if etcd.IsBeingProcessed() { + // allow operations on resources if any etcd operation is currently being reconciled, but only by etcd-druid, + // and allow exempt service accounts to make changes to resources, but only if etcd is not currently being reconciled. + if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { - return admission.Allowed(fmt.Sprintf("ongoing processing of etcd %s by etcd-druid requires changes to resources", etcd.Name)) + return admission.Allowed(fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", etcd.Name)) } - return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing processing of etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", etcd.Name)) } else { for _, sa := range h.config.ExemptServiceAccounts { if req.UserInfo.Username == sa { From c5ac2f06676a15bbe790f0738ec8a44b615aff3b Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 16 Feb 2024 14:01:19 +0530 Subject: [PATCH 093/235] Sentinel to freely allow updates on lease resources --- internal/webhook/sentinel/handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 1bccf3b56..06e86dea6 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -80,6 +80,9 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R case batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(): obj, err = h.decodeJob(req, decodeOldObject) case coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): + if req.Operation == admissionv1.Update { + return admission.Allowed("lease resource can be freely updated") + } obj, err = h.decodeLease(req, decodeOldObject) default: return admission.Allowed(fmt.Sprintf("unexpected resource type: %s", requestGK)) From 0635876d5a4d05b5126fd88a06b1747c34e16414 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 21 Feb 2024 13:13:13 +0530 Subject: [PATCH 094/235] changed register for etcd controller and adapted unit tests --- internal/controller/etcd/register.go | 108 ++++++---- internal/controller/etcd/register_test.go | 239 ++++++++++++++++++---- 2 files changed, 271 insertions(+), 76 deletions(-) diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index c4d7effb5..94d6e2d00 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -8,7 +8,6 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -31,21 +30,84 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { return builder.Complete(r) } +// buildPredicate returns a predicate that filters out events that are not relevant for the Etcd controller. +// NOTE: +// For all conditions the following is applicable: +// 1. create and delete events are always reconciled. +// 2. generic events are not reconciled. If there is a need in future to react to generic events then this should be changed. +// Conditions for reconciliation: +// Scenario 1: {Auto-Reconcile: false, Reconcile-Annotation-Present: false, Spec-Updated: false/true, Status-Updated: false/true, update-event-reconciled: false} +// Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} +// Scenario 3: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} +// Scenario 4: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} +// Scenario 5: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} +// Scenario 6: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: NA}, This condition cannot happen. In case of a controller restart there will only be a CreateEvent. +// Scenario 7: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} +// Scenario 8: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} +// Scenario 9: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} +// Scenario 10: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: false} +// Scenario 11: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} +// Scenario 12: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} +// Scenario 13: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} +// Scenario 14: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} func (r *Reconciler) buildPredicate() predicate.Predicate { - // Since we do etcd status updates in the same reconciliation flow, this will also generate an event which should be ignored. Any changes in the predicates should ensure that this is not violated. + /* + If there is no change to spec and status then no reconciliation would happen. This is also true when auto-reconcile + has been enabled. If an operator wishes to force a reconcile especially when no change (spec/status) has been done to the etcd resource + then the only way is to explicitly add the reconcile annotation to the etcd resource. + */ + forceReconcilePredicate := predicate.And( + r.hasReconcileAnnotation(), + noSpecAndStatusUpdated(), + ) + /* + If there is a spec change (irrespective of status change) and if there is an update event then it will trigger a reconcile only when either + auto-reconcile has been enabled or an operator has added the reconcile annotation to the etcd resource. + */ + onSpecChangePredicate := predicate.And( + predicate.Or( + r.hasReconcileAnnotation(), + r.autoReconcileEnabled(), + ), + specUpdated(), + ) + return predicate.Or( - // If reconcile operation annotation is present but there is no change to spec or status then we should allow reconciliation. Consider a case where the spec reconciliation has failed - predicate.And(r.forcedReconcile(), noSpecAndStatusUpdated()), - // If the reconciliation is allowed and the spec has changed, we should reconcile. - predicate.And(r.reconcilePermitted(), onlySpecUpdated()), + forceReconcilePredicate, + onSpecChangePredicate, ) } -func onlySpecUpdated() predicate.Predicate { +func (r *Reconciler) hasReconcileAnnotation() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return updateEvent.ObjectNew.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile + }, + CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, + GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, + } +} + +func (r *Reconciler) autoReconcileEnabled() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation + }, + CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, + GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, + } +} + +func specUpdated() predicate.Predicate { return predicate.Funcs{ UpdateFunc: func(updateEvent event.UpdateEvent) bool { - return hasSpecChanged(updateEvent) && !hasStatusChanged(updateEvent) + return hasSpecChanged(updateEvent) }, + CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, + GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, } } @@ -54,6 +116,9 @@ func noSpecAndStatusUpdated() predicate.Predicate { UpdateFunc: func(updateEvent event.UpdateEvent) bool { return !hasSpecChanged(updateEvent) && !hasStatusChanged(updateEvent) }, + GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, + CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, } } @@ -72,30 +137,3 @@ func hasStatusChanged(updateEvent event.UpdateEvent) bool { } return !apiequality.Semantic.DeepEqual(oldEtcd.Status, newEtcd.Status) } - -// reconcilePermitted -func (r *Reconciler) reconcilePermitted() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(updateEvent event.UpdateEvent) bool { - if r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation { - return true - } - return !hasReconcileAnnotation(updateEvent.ObjectOld) && hasReconcileAnnotation(updateEvent.ObjectNew) - }, - } -} - -func (r *Reconciler) forcedReconcile() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(updateEvent event.UpdateEvent) bool { - if r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation { - return false - } - return !hasReconcileAnnotation(updateEvent.ObjectOld) && hasReconcileAnnotation(updateEvent.ObjectNew) - }, - } -} - -func hasReconcileAnnotation(object client.Object) bool { - return object.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile -} diff --git a/internal/controller/etcd/register_test.go b/internal/controller/etcd/register_test.go index b5fa38136..a9fe3cd96 100644 --- a/internal/controller/etcd/register_test.go +++ b/internal/controller/etcd/register_test.go @@ -4,6 +4,7 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + mockmanager "github.com/gardener/etcd-druid/internal/mock/controller-runtime/manager" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/golang/mock/gomock" @@ -11,82 +12,239 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" - - mockmanager "github.com/gardener/etcd-druid/internal/mock/controller-runtime/manager" ) -type preCondition struct { - enableEtcdSpecAutoReconcile bool - ignoreOperationAnnotation bool - reconcileAnnotationPresent bool - etcdSpecChanged bool - etcdStatusChanged bool +func TestBuildPredicateWithOnlyAutoReconcileEnabled(t *testing.T) { + testCases := []struct { + name string + etcdSpecChanged bool + etcdStatusChanged bool + // expected behavior for different event types + shouldAllowCreateEvent bool + shouldAllowDeleteEvent bool + shouldAllowGenericEvent bool + shouldAllowUpdateEvent bool + }{ + { + name: "only spec has changed", + etcdSpecChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "only status has changed", + etcdStatusChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "both spec and status have changed", + etcdSpecChanged: true, + etcdStatusChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "neither spec nor status has changed", + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + } + g := NewWithT(t) + etcd := createEtcd() + r := createReconciler(t, true) + predicate := r.buildPredicate() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, false) + g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) + g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) + g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) + g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: etcd, ObjectNew: updatedEtcd})).To(Equal(tc.shouldAllowUpdateEvent)) + }) + } } -func TestBuildPredicate(t *testing.T) { +func TestBuildPredicateWithNoAutoReconcileAndNoReconcileAnnot(t *testing.T) { testCases := []struct { - name string - preCondition preCondition + name string + etcdSpecChanged bool + etcdStatusChanged bool // expected behavior for different event types shouldAllowCreateEvent bool shouldAllowDeleteEvent bool shouldAllowGenericEvent bool shouldAllowUpdateEvent bool }{ - // TODO (madhav): remove this test case once ignoreOperationAnnotation has been removed. It has already been marked as deprecated. { - name: "when ignoreOperationAnnotation is true and only spec has changed", - preCondition: preCondition{ignoreOperationAnnotation: true, reconcileAnnotationPresent: false, etcdSpecChanged: true}, + name: "only spec has changed", + etcdSpecChanged: true, shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, - shouldAllowGenericEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "only status has changed", + etcdStatusChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "both spec and status have changed", + etcdSpecChanged: true, + etcdStatusChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "neither spec nor status has changed", + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + } + g := NewWithT(t) + etcd := createEtcd() + r := createReconciler(t, false) + predicate := r.buildPredicate() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, false) + g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) + g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) + g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) + g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: etcd, ObjectNew: updatedEtcd})).To(Equal(tc.shouldAllowUpdateEvent)) + }) + } +} + +func TestBuildPredicateWithNoAutoReconcileButReconcileAnnotPresent(t *testing.T) { + testCases := []struct { + name string + etcdSpecChanged bool + etcdStatusChanged bool + // expected behavior for different event types + shouldAllowCreateEvent bool + shouldAllowDeleteEvent bool + shouldAllowGenericEvent bool + shouldAllowUpdateEvent bool + }{ + { + name: "only spec has changed", + etcdSpecChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, shouldAllowUpdateEvent: true, }, - // TODO (madhav): remove this test case once ignoreOperationAnnotation has been removed. It has already been marked as deprecated. { - name: "when ignoreOperationAnnotation is true and there is no change to spec but only to status", - preCondition: preCondition{ignoreOperationAnnotation: true, reconcileAnnotationPresent: false, etcdStatusChanged: true}, + name: "only status has changed", + etcdStatusChanged: true, shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, - shouldAllowGenericEvent: true, + shouldAllowGenericEvent: false, shouldAllowUpdateEvent: false, }, { - name: "when enableEtcdSpecAutoReconcile is true and only spec has changed", - preCondition: preCondition{enableEtcdSpecAutoReconcile: true, reconcileAnnotationPresent: false, etcdSpecChanged: true}, + name: "both spec and status have changed", + etcdSpecChanged: true, + etcdStatusChanged: true, shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, - shouldAllowGenericEvent: true, + shouldAllowGenericEvent: false, shouldAllowUpdateEvent: true, }, { - name: "when enableEtcdSpecAutoReconcile is true and there is no change to spec but only to status", - preCondition: preCondition{enableEtcdSpecAutoReconcile: true, reconcileAnnotationPresent: false, etcdStatusChanged: true}, + name: "neither spec nor status has changed", shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, - shouldAllowGenericEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + } + g := NewWithT(t) + etcd := createEtcd() + r := createReconciler(t, false) + predicate := r.buildPredicate() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, true) + g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) + g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) + g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) + g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: etcd, ObjectNew: updatedEtcd})).To(Equal(tc.shouldAllowUpdateEvent)) + }) + } +} + +func TestBuildPredicateWithAutoReconcileAndReconcileAnnotSet(t *testing.T) { + testCases := []struct { + name string + etcdSpecChanged bool + etcdStatusChanged bool + // expected behavior for different event types + shouldAllowCreateEvent bool + shouldAllowDeleteEvent bool + shouldAllowGenericEvent bool + shouldAllowUpdateEvent bool + }{ + { + name: "only spec has changed", + etcdSpecChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "only status has changed", + etcdStatusChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, shouldAllowUpdateEvent: false, }, { - name: "when reconcile annotation is present and no change to spec and status", - preCondition: preCondition{reconcileAnnotationPresent: true}, + name: "both spec and status have changed", + etcdSpecChanged: true, + etcdStatusChanged: true, + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "neither spec nor status has changed", shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, - shouldAllowGenericEvent: true, + shouldAllowGenericEvent: false, shouldAllowUpdateEvent: true, }, } g := NewWithT(t) etcd := createEtcd() - etcd.Status.Replicas = 1 + r := createReconciler(t, true) + predicate := r.buildPredicate() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - r := createReconciler(t, tc.preCondition.enableEtcdSpecAutoReconcile, tc.preCondition.ignoreOperationAnnotation) - predicate := r.buildPredicate() - g.Expect(predicate.Create(event.CreateEvent{Object: etcd})).To(Equal(tc.shouldAllowCreateEvent)) - g.Expect(predicate.Delete(event.DeleteEvent{Object: etcd})).To(Equal(tc.shouldAllowDeleteEvent)) - g.Expect(predicate.Generic(event.GenericEvent{Object: etcd})).To(Equal(tc.shouldAllowGenericEvent)) - updatedEtcd := updateEtcd(tc.preCondition, etcd) + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, true) + g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) + g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) + g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: etcd, ObjectNew: updatedEtcd})).To(Equal(tc.shouldAllowUpdateEvent)) }) } @@ -109,19 +267,19 @@ func createEtcd() *druidv1alpha1.Etcd { return etcd } -func updateEtcd(preCondition preCondition, originalEtcd *druidv1alpha1.Etcd) *druidv1alpha1.Etcd { +func updateEtcd(originalEtcd *druidv1alpha1.Etcd, specChanged, statusChanged, reconcileAnnotPresent bool) *druidv1alpha1.Etcd { newEtcd := originalEtcd.DeepCopy() annotations := make(map[string]string) - if preCondition.reconcileAnnotationPresent { + if reconcileAnnotPresent { annotations[v1beta1constants.GardenerOperation] = v1beta1constants.GardenerOperationReconcile newEtcd.SetAnnotations(annotations) } - if preCondition.etcdSpecChanged { + if specChanged { // made a single change to the spec newEtcd.Spec.Backup.Image = pointer.String("eu.gcr.io/gardener-project/gardener/etcdbrctl-distroless:v1.0.0") newEtcd.Generation++ } - if preCondition.etcdStatusChanged { + if statusChanged { // made a single change to the status newEtcd.Status.ReadyReplicas = 2 newEtcd.Status.Ready = pointer.Bool(false) @@ -129,7 +287,7 @@ func updateEtcd(preCondition preCondition, originalEtcd *druidv1alpha1.Etcd) *dr return newEtcd } -func createReconciler(t *testing.T, enableEtcdSpecAutoReconcile, ignoreOperationAnnotation bool) *Reconciler { +func createReconciler(t *testing.T, enableEtcdSpecAutoReconcile bool) *Reconciler { g := NewWithT(t) mockCtrl := gomock.NewController(t) mgr := mockmanager.NewMockManager(mockCtrl) @@ -137,7 +295,6 @@ func createReconciler(t *testing.T, enableEtcdSpecAutoReconcile, ignoreOperation mgr.EXPECT().GetClient().AnyTimes().Return(testutils.NewTestClientBuilder().WithClient(fakeClient).Build()) mgr.EXPECT().GetEventRecorderFor(gomock.Any()).AnyTimes().Return(nil) etcdConfig := Config{ - IgnoreOperationAnnotation: ignoreOperationAnnotation, EnableEtcdSpecAutoReconcile: enableEtcdSpecAutoReconcile, } r, err := NewReconcilerWithImageVector(mgr, &etcdConfig, nil) From 63d5d0749d3365d3f768cf9c55c6e5be7ccdf5d8 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 4 Mar 2024 11:41:46 +0530 Subject: [PATCH 095/235] moved to golang version 1.22 --- .ci/hack/component_descriptor | 0 .ci/hack/prepare_release | 0 .ci/hack/set_dependency_version | 0 .ci/pipeline_definitions | 8 ++++---- Dockerfile | 2 +- go.mod | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) mode change 100755 => 100644 .ci/hack/component_descriptor mode change 100755 => 100644 .ci/hack/prepare_release mode change 100755 => 100644 .ci/hack/set_dependency_version diff --git a/.ci/hack/component_descriptor b/.ci/hack/component_descriptor old mode 100755 new mode 100644 diff --git a/.ci/hack/prepare_release b/.ci/hack/prepare_release old mode 100755 new mode 100644 diff --git a/.ci/hack/set_dependency_version b/.ci/hack/set_dependency_version old mode 100755 new mode 100644 diff --git a/.ci/pipeline_definitions b/.ci/pipeline_definitions index 71f5a990d..cebb90752 100644 --- a/.ci/pipeline_definitions +++ b/.ci/pipeline_definitions @@ -35,13 +35,13 @@ etcd-druid: teamname: 'gardener/etcd-druid-maintainers' steps: check: - image: 'golang:1.21.4' + image: 'golang:1.22.0' test: - image: 'golang:1.21.4' + image: 'golang:1.22.0' test_integration: - image: 'golang:1.21.4' + image: 'golang:1.22.0' build: - image: 'golang:1.21.4' + image: 'golang:1.22.0' output_dir: 'binary' jobs: diff --git a/Dockerfile b/Dockerfile index 94a9e3178..767974bd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.21.4 as builder +FROM golang:1.22.0 as builder WORKDIR /go/src/github.com/gardener/etcd-druid COPY . . diff --git a/go.mod b/go.mod index 3b4fb5823..59cd12e78 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gardener/etcd-druid -go 1.21 +go 1.22 require ( github.com/gardener/etcd-backup-restore v0.26.0 From 96d728ab6ccd952c5c8ff0d5ca9f1b21f7468db4 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 14 Mar 2024 12:56:02 +0530 Subject: [PATCH 096/235] renamed LastError.LastUpdateTime to ObservedAt, introduced constants when building sts --- api/v1alpha1/types_etcd.go | 7 +- api/v1alpha1/zz_generated.deepcopy.go | 2 +- internal/errors/errors.go | 17 ++-- internal/operator/statefulset/builder.go | 79 +++++++++++++++---- internal/operator/statefulset/constants.go | 34 ++++++++ internal/operator/statefulset/statefulset.go | 83 ++++++++++---------- 6 files changed, 155 insertions(+), 67 deletions(-) create mode 100644 internal/operator/statefulset/constants.go diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index a2f5ab7ec..1607ed6c3 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -390,6 +390,7 @@ type EtcdStatus struct { // +optional LastErrors []LastError `json:"lastErrors,omitempty"` // LastOperation indicates the last operation performed on this resource. + // +optional LastOperation *LastOperation `json:"lastOperation,omitempty"` // Cluster size is the current size of the etcd cluster. // Deprecated: this field will not be populated with any value and will be removed in the future. @@ -461,7 +462,7 @@ type LastOperation struct { // generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this // as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering // reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. - RunID string `json:"rundID"` + RunID string `json:"runID"` // LastUpdateTime is the time at which the operation was updated. LastUpdateTime metav1.Time `json:"lastUpdateTime"` } @@ -475,8 +476,8 @@ type LastError struct { Code ErrorCode `json:"code"` // Description is a human-readable message indicating details of the error. Description string `json:"description"` - // LastUpdateTime is the time the error was reported. - LastUpdateTime metav1.Time `json:"lastUpdateTime"` + // ObservedAt is the time the error was observed. + ObservedAt metav1.Time `json:"lastUpdateTime"` } // GetNamespaceName is a convenience function which creates a types.NamespacedName for an etcd resource. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ab4a188ba..8fd1cb4c3 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -630,7 +630,7 @@ func (in *EtcdStatus) DeepCopy() *EtcdStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LastError) DeepCopyInto(out *LastError) { *out = *in - in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.ObservedAt.DeepCopyInto(&out.ObservedAt) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LastError. diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 76f18567a..781b1b858 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -21,6 +21,8 @@ type DruidError struct { Operation string // Message is the custom message providing additional context for the error. Message string + // ObservedAt is the time at which the error was observed. + ObservedAt time.Time } func (r *DruidError) Error() string { @@ -34,10 +36,11 @@ func WrapError(err error, code druidv1alpha1.ErrorCode, operation string, messag return nil } return &DruidError{ - Code: code, - Cause: err, - Operation: operation, - Message: message, + Code: code, + Cause: err, + Operation: operation, + Message: message, + ObservedAt: time.Now().UTC(), } } @@ -50,9 +53,9 @@ func MapToLastErrors(errs []error) []druidv1alpha1.LastError { if errors.As(err, &druidErr) { desc := fmt.Sprintf("[Operation: %s, Code: %s] message: %s, cause: %s", druidErr.Operation, druidErr.Code, druidErr.Message, druidErr.Cause.Error()) lastErr := druidv1alpha1.LastError{ - Code: druidErr.Code, - Description: desc, - LastUpdateTime: metav1.NewTime(time.Now().UTC()), + Code: druidErr.Code, + Description: desc, + ObservedAt: metav1.NewTime(druidErr.ObservedAt), } lastErrs = append(lastErrs, lastErr) } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 9e1941d31..e564076f4 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -256,15 +256,14 @@ func (b *stsBuilder) getEtcdContainerVolumeMounts() []corev1.VolumeMount { func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMount { brVolumeMounts := make([]corev1.VolumeMount, 0, 8) - brVolumeMounts = append(brVolumeMounts, b.getEtcdDataVolumeMount(), corev1.VolumeMount{ - Name: "etcd-config-file", - MountPath: "/var/etcd/config/", + Name: etcdConfigFileName, + MountPath: etcdConfigFileMountPath, }, ) - brVolumeMounts = append(brVolumeMounts, b.getSecretVolumeMounts()...) + brVolumeMounts = append(brVolumeMounts, b.getBackupRestoreSecretVolumeMounts()...) if b.etcd.IsBackupStoreEnabled() { etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() @@ -276,6 +275,23 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun return brVolumeMounts } +func (b *stsBuilder) getBackupRestoreSecretVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: backRestoreCAVolumeName, + MountPath: backupRestoreCAVolumeMountPath, + }, + { + Name: backRestoreServerTLSVolumeName, + MountPath: backupRestoreServerTLSVolumeMountPath, + }, + { + Name: backRestoreClientTLSVolumeName, + MountPath: backupRestoreClientTLSVolumeMountPath, + }, + } +} + func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { switch *b.provider { case utils.Local: @@ -310,7 +326,7 @@ func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) return corev1.VolumeMount{ Name: volumeClaimTemplateName, - MountPath: "/var/etcd/data", + MountPath: etcdDataVolumeMountPath, } } @@ -421,8 +437,8 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) } if b.etcd.Spec.Backup.TLS != nil { - commandArgs = append(commandArgs, "--server-cert=/var/etcd/ssl/client/server/tls.crt") - commandArgs = append(commandArgs, "--server-key=/var/etcd/ssl/client/server/tls.key") + commandArgs = append(commandArgs, fmt.Sprintf("--server-cert=%s/tls.crt", backupRestoreServerTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--server-key=%s/tls.key", backupRestoreServerTLSVolumeMountPath)) } // Other misc command line args @@ -652,7 +668,7 @@ func (b *stsBuilder) getSecretVolumeMounts() []corev1.VolumeMount { func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volume, error) { volumes := []corev1.Volume{ { - Name: "etcd-config-file", + Name: etcdConfigVolumeName, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ @@ -660,8 +676,8 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu }, Items: []corev1.KeyToPath{ { - Key: "etcd.conf.yaml", - Path: "etcd.conf.yaml", + Key: etcdConfigFileName, + Path: etcdConfigFileName, }, }, DefaultMode: pointer.Int32(0644), @@ -676,6 +692,9 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu if b.etcd.Spec.Etcd.PeerUrlTLS != nil { volumes = append(volumes, b.getPeerTLSVolumes()...) } + if b.etcd.Spec.Backup.TLS != nil { + volumes = append(volumes, b.getBackupRestoreTLSVolumes()...) + } if b.etcd.IsBackupStoreEnabled() { backupVolume, err := b.getBackupVolume(ctx) if err != nil { @@ -692,7 +711,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { clientTLSConfig := b.etcd.Spec.Etcd.ClientUrlTLS return []corev1.Volume{ { - Name: "client-url-ca-etcd", + Name: clientCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.TLSCASecretRef.Name, @@ -700,7 +719,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { }, }, { - Name: "client-url-etcd-server-tls", + Name: serverTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ServerTLSSecretRef.Name, @@ -708,7 +727,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { }, }, { - Name: "client-url-etcd-client-tls", + Name: clientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ClientTLSSecretRef.Name, @@ -722,7 +741,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { peerTLSConfig := b.etcd.Spec.Etcd.PeerUrlTLS return []corev1.Volume{ { - Name: "peer-url-ca-etcd", + Name: peerCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.TLSCASecretRef.Name, @@ -730,7 +749,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { }, }, { - Name: "peer-url-etcd-server-tls", + Name: peerServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.ServerTLSSecretRef.Name, @@ -740,6 +759,36 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { } } +func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { + tlsConfig := b.etcd.Spec.Backup.TLS + return []corev1.Volume{ + { + Name: backRestoreCAVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsConfig.TLSCASecretRef.Name, + }, + }, + }, + { + Name: backRestoreServerTLSVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsConfig.ServerTLSSecretRef.Name, + }, + }, + }, + { + Name: backRestoreClientTLSVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsConfig.ClientTLSSecretRef.Name, + }, + }, + }, + } +} + func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Volume, error) { if b.provider == nil { return nil, nil diff --git a/internal/operator/statefulset/constants.go b/internal/operator/statefulset/constants.go new file mode 100644 index 000000000..8a96f1f15 --- /dev/null +++ b/internal/operator/statefulset/constants.go @@ -0,0 +1,34 @@ +package statefulset + +// constants for volume names +const ( + // clientCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. + clientCAVolumeName = "ca" + // serverTLSVolumeName is the name of the volume that contains the server certificate used to set up the server (etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). + serverTLSVolumeName = "server-tls" + // clientTLSVolumeName is the name of the volume that contains the client certificate used by the client to communicate to the server(etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). + clientTLSVolumeName = "client-tls" + peerCAVolumeName = "peer-ca" + peerServerTLSVolumeName = "peer-server-tls" + backRestoreCAVolumeName = "back-restore-ca" + backRestoreServerTLSVolumeName = "back-restore-server-tls" + backRestoreClientTLSVolumeName = "back-restore-client-tls" + etcdConfigVolumeName = "etcd-config-file" +) + +// constants for volume mount paths +const ( + backupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" + backupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" + backupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" +) + +const ( + etcdConfigFileName = "etcd.conf.yaml" + etcdConfigFileMountPath = "/var/etcd/config/" +) + +const ( + // etcdDataVolumeMountPath is the path on etcd and etcd-backup-restore containers where etcd data directory is hosted. + etcdDataVolumeMountPath = "/var/etcd/data" +) diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 1f4358bcf..6e886ccf5 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -69,12 +69,12 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ErrSyncStatefulSet, "Sync", fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) - } // There is no StatefulSet present. Create one. if existingSTS == nil { return r.createOrPatch(ctx, etcd) } + // StatefulSet exists, check if TLS has been enabled for peer communication, if yes then it is currently a multistep // process to ensure that all members are updated and establish peer TLS communication. if err = r.handlePeerTLSChanges(ctx, etcd, existingSTS); err != nil { @@ -152,53 +152,54 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) } - /** - Case 1: - Scaled from 1 to 3 and peer URL TLS changed enabled or disabled - g/g case - Case 2: - Scaled from 1 to 3 and no change in peer URL TLS (can be kept as HTTP or HTTPS) - Case 3: - Peer URL TLS changed but no change in replicas - */ - - /** - After the changes are made to etcd-wrapper and etcd-backup-restore then we can use the following logic and simplify the handling of peer URL TLS change - if peerTLSNotEnabledOnAllMembers { - if sts.HasAnnotationValue(etcdGenerationAnnotation) != etcd.Spec.Generation { - createOrPathKeepingExistingStsReplicas + if isPeerTLSChangedToEnabled(peerTLSEnabledForAllMembers, etcd) { + if !isStatefulSetPatchedWithPeerTLSVolMount(existingSts) { + // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd component due to + // enabling of TLS for peer communication. It preserves the current STS replicas. + if err = r.createOrPatchWithReplicas(ctx, etcd, *existingSts.Spec.Replicas); err != nil { + return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %w", err) + } + } else { + ctx.Logger.Info("Secret volume mounts to enable Peer URL TLS have already been mounted. Skipping patching StatefulSet with secret volume mounts.") } - if !sts.Ready() { - requeue + // check again if peer TLS has been enabled for all members. If not then force a requeue of the reconcile request. + if !peerTLSEnabledForAllMembers { + return fmt.Errorf("peer URL TLS not enabled for all members for etcd: %v, requeuing reconcile request", etcd.GetNamespaceName()) } - } - */ - - if isPeerTLSChangedToEnabled(peerTLSEnabledForAllMembers, etcd) { - ctx.Logger.Info("Attempting to enable TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) - // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd component due to - // enabling of TLS for peer communication. It preserves the current STS replicas. - if err = r.createOrPatchWithReplicas(ctx, etcd, *existingSts.Spec.Replicas); err != nil { - return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %w", err) - } + //ctx.Logger.Info("Attempting to enable TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) + // + //tlsEnabled, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) + //if err != nil { + // return fmt.Errorf("error verifying if TLS is enabled post-patch: %v", err) + //} + //// It usually takes time for TLS to be enabled and reflected via the lease. So first time around this will not be true. + //// So instead of waiting we requeue the request to be re-tried again. + //if !tlsEnabled { + // return fmt.Errorf("failed to enable TLS for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) + //} + // + //if err := deleteAllStsPods(ctx, r.client, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { + // return fmt.Errorf("error deleting StatefulSet pods after enabling TLS: %v", err) + //} + //ctx.Logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) + } + ctx.Logger.Info("Peer URL TLS has been enabled for all members") + return nil +} - tlsEnabled, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) - if err != nil { - return fmt.Errorf("error verifying if TLS is enabled post-patch: %v", err) +func isStatefulSetPatchedWithPeerTLSVolMount(existingSts *appsv1.StatefulSet) bool { + volumes := existingSts.Spec.Template.Spec.Volumes + var peerURLCAEtcdVolPresent, peerURLEtcdServerTLSVolPresent bool + for _, vol := range volumes { + if vol.Name == "peer-url-ca-etcd" { + peerURLCAEtcdVolPresent = true } - // It usually takes time for TLS to be enabled and reflected via the lease. So first time around this will not be true. - // So instead of waiting we requeue the request to be re-tried again. - if !tlsEnabled { - return fmt.Errorf("failed to enable TLS for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) + if vol.Name == "peer-url-etcd-server-tls" { + peerURLEtcdServerTLSVolPresent = true } - - if err := deleteAllStsPods(ctx, r.client, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { - return fmt.Errorf("error deleting StatefulSet pods after enabling TLS: %v", err) - } - ctx.Logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) } - - return nil + return peerURLCAEtcdVolPresent && peerURLEtcdServerTLSVolPresent } // isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled From 76a0fe19dc975d5d75bf3ac7c6609a446affbc72 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 14 Mar 2024 13:56:20 +0530 Subject: [PATCH 097/235] Fix and run `make generate` --- .ci/hack/component_descriptor | 0 .ci/hack/prepare_release | 0 .ci/hack/set_dependency_version | 0 api/v1alpha1/types_etcd.go | 2 +- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 27 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 1270 ++++++++--------- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 27 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 1270 ++++++++--------- hack/tools.mk | 2 +- 9 files changed, 1276 insertions(+), 1322 deletions(-) mode change 100644 => 100755 .ci/hack/component_descriptor mode change 100644 => 100755 .ci/hack/prepare_release mode change 100644 => 100755 .ci/hack/set_dependency_version diff --git a/.ci/hack/component_descriptor b/.ci/hack/component_descriptor old mode 100644 new mode 100755 diff --git a/.ci/hack/prepare_release b/.ci/hack/prepare_release old mode 100644 new mode 100755 diff --git a/.ci/hack/set_dependency_version b/.ci/hack/set_dependency_version old mode 100644 new mode 100755 diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 1607ed6c3..23c87198d 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -477,7 +477,7 @@ type LastError struct { // Description is a human-readable message indicating details of the error. Description string `json:"description"` // ObservedAt is the time the error was observed. - ObservedAt metav1.Time `json:"lastUpdateTime"` + ObservedAt metav1.Time `json:"observedAt"` } // GetNamespaceName is a convenience function which creates a types.NamespacedName for an etcd resource. diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 16951ef91..9aa0b085e 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -25,14 +25,19 @@ spec: source to a target store. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -41,9 +46,9 @@ spec: backups task. properties: maxBackupAge: - description: MaxBackupAge is the maximum age in days that a backup - must have in order to be copied. By default all backups will be - copied. + description: |- + MaxBackupAge is the maximum age in days that a backup must have in order to be copied. + By default all backups will be copied. format: int32 type: integer maxBackups: @@ -122,8 +127,8 @@ spec: snapshot before copying backups. type: boolean timeout: - description: Timeout is the timeout for waiting for a final full - snapshot. When this timeout expires, the copying of backups + description: |- + Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups will be performed anyway. No timeout or 0 means wait forever. type: string required: diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index cdbce78a8..cfccfd992 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -48,14 +48,19 @@ spec: description: Etcd is the Schema for the etcds API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -71,23 +76,29 @@ spec: and delta snapshots of etcd. properties: compactionResources: - description: 'CompactionResources defines compute Resources required - by compaction job. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: |- + CompactionResources defines compute Resources required by compaction job. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ properties: claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string required: - name @@ -103,8 +114,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -113,11 +125,19 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true +<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' +======= + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +>>>>>>> 8571402f (Fix and run `make generate`) type: object type: object compression: @@ -148,10 +168,9 @@ spec: delta snapshots will be taken type: string deltaSnapshotRetentionPeriod: - description: DeltaSnapshotRetentionPeriod defines the duration - for which delta snapshots will be retained, excluding the latest - snapshot set. The value should be a string formatted as a duration - (e.g., '1s', '2m', '3h', '4d') + description: |- + DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set. + The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d') pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ type: string enableProfiling: @@ -205,23 +224,29 @@ spec: format: int32 type: integer resources: - description: 'Resources defines compute Resources required by - backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: |- + Resources defines compute Resources required by backup-restore container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ properties: claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string required: - name @@ -237,8 +262,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -247,11 +273,19 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true +<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' +======= + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +>>>>>>> 8571402f (Fix and run `make generate`) type: object type: object store: @@ -289,8 +323,9 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -303,8 +338,9 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -342,8 +378,9 @@ spec: description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: - description: SecretReference represents a Secret Reference. It - has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -380,8 +417,9 @@ spec: TLS secrets for client communication to ETCD cluster properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -394,8 +432,9 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -451,14 +490,14 @@ spec: - extensive type: string peerUrlTls: - description: PeerUrlTLS contains the ca and server TLS secrets - for peer communication within ETCD cluster Currently, PeerUrlTLS - does not require client TLS secrets for gardener implementation - of ETCD cluster. + description: |- + PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster + Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -471,8 +510,9 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -513,23 +553,29 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resources: - description: 'Resources defines the compute Resources required - by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: |- + Resources defines the compute Resources required by etcd container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ properties: claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string required: - name @@ -545,8 +591,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -555,11 +602,19 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true +<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' +======= + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +>>>>>>> 8571402f (Fix and run `make generate`) type: object type: object serverPort: @@ -578,35 +633,35 @@ spec: format: int32 type: integer schedulingConstraints: - description: SchedulingConstraints defines the different scheduling - constraints that must be applied to the pod spec in the etcd statefulset. + description: |- + SchedulingConstraints defines the different scheduling constraints that must be applied to the + pod spec in the etcd statefulset. Currently supported constraints are Affinity and TopologySpreadConstraints. properties: affinity: - description: Affinity defines the various affinity and anti-affinity - rules for a pod that are honoured by the kube-scheduler. + description: |- + Affinity defines the various affinity and anti-affinity rules for a pod + that are honoured by the kube-scheduler. properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: description: A node selector term, associated with @@ -616,32 +671,26 @@ spec: description: A list of node selector requirements by node's labels. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -654,32 +703,26 @@ spec: description: A list of node selector requirements by node's fields. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -701,53 +744,46 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -760,32 +796,26 @@ spec: description: A list of node selector requirements by node's fields. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -808,18 +838,16 @@ spec: other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred @@ -838,30 +866,25 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -873,53 +896,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by - this field and the ones listed in the namespaces - field. null selector and null or empty namespaces - list means "this pod's namespace". An empty - selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -931,42 +946,37 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. - The term is applied to the union of the namespaces - listed in this field and the ones selected - by namespaceSelector. null or empty namespaces - list and null namespaceSelector means "this - pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -975,23 +985,22 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, @@ -1002,28 +1011,24 @@ spec: selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1036,51 +1041,44 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by this - field and the ones listed in the namespaces field. - null selector and null or empty namespaces list - means "this pod's namespace". An empty selector - ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1093,33 +1091,29 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. The - term is applied to the union of the namespaces - listed in this field and the ones selected by - namespaceSelector. null or empty namespaces list - and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -1132,18 +1126,16 @@ spec: as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred @@ -1162,30 +1154,25 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -1197,53 +1184,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by - this field and the ones listed in the namespaces - field. null selector and null or empty namespaces - list means "this pod's namespace". An empty - selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -1255,42 +1234,37 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. - The term is applied to the union of the namespaces - listed in this field and the ones selected - by namespaceSelector. null or empty namespaces - list and null namespaceSelector means "this - pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -1299,23 +1273,22 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, @@ -1326,28 +1299,24 @@ spec: selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1360,51 +1329,44 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by this - field and the ones listed in the namespaces field. - null selector and null or empty namespaces list - means "this pod's namespace". An empty selector - ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1417,33 +1379,29 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. The - term is applied to the union of the namespaces - listed in this field and the ones selected by - namespaceSelector. null or empty namespaces list - and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -1452,24 +1410,25 @@ spec: type: object type: object topologySpreadConstraints: - description: TopologySpreadConstraints describes how a group of - pods ought to spread across topology domains, that are honoured - by the kube-scheduler. + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology domains, + that are honoured by the kube-scheduler. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: - description: LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine - the number of pods in their corresponding topology domain. + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: @@ -1477,17 +1436,16 @@ spec: applies to. type: string operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, - NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists - or DoesNotExist, the values array must be empty. - This array is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -1499,15 +1457,15 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field - is "key", the operator is "In", and the values array - contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: +<<<<<<< HEAD description: "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod @@ -1521,110 +1479,123 @@ spec: against labelSelector. \n This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default)." +======= + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. +>>>>>>> 8571402f (Fix and run `make generate`) items: type: string type: array x-kubernetes-list-type: atomic maxSkew: - description: 'MaxSkew describes the degree to which pods - may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, - it is the maximum permitted difference between the number - of matching pods in the target topology and the global - minimum. The global minimum is the minimum number of matching - pods in an eligible domain or zero if the number of eligible - domains is less than MinDomains. For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with the same labelSelector - spread as 2/2/1: In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to - zone3 to become 2/2/2; scheduling it onto zone1(zone2) - would make the ActualSkew(3-1) on zone1(zone2) violate - MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled - onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, - it is used to give higher precedence to topologies that - satisfy it. It''s a required field. Default value is 1 - and 0 is not allowed.' + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. format: int32 type: integer minDomains: - description: "MinDomains indicates a minimum number of eligible - domains. When the number of eligible domains with matching - topology keys is less than minDomains, Pod Topology Spread - treats \"global minimum\" as 0, and then the calculation - of Skew is performed. And when the number of eligible - domains with matching topology keys equals or greater - than minDomains, this value has no effect on scheduling. - As a result, when the number of eligible domains is less - than minDomains, scheduler won't schedule more than maxSkew - Pods to those domains. If value is nil, the constraint - behaves as if MinDomains is equal to 1. Valid values are - integers greater than 0. When value is not nil, WhenUnsatisfiable - must be DoNotSchedule. \n For example, in a 3-zone cluster, - MaxSkew is set to 2, MinDomains is set to 5 and pods with - the same labelSelector spread as 2/2/2: | zone1 | zone2 - | zone3 | | P P | P P | P P | The number of domains - is less than 5(MinDomains), so \"global minimum\" is treated - as 0. In this situation, new pod with the same labelSelector - cannot be scheduled, because computed skew will be 3(3 - - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. \n This is a beta field and requires - the MinDomainsInPodTopologySpread feature gate to be enabled - (enabled by default)." + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: - description: "NodeAffinityPolicy indicates how we will treat - Pod's nodeAffinity/nodeSelector when calculating pod topology - spread skew. Options are: - Honor: only nodes matching - nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes - are included in the calculations. \n If this value is - nil, the behavior is equivalent to the Honor policy. This - is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread - feature flag." + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string nodeTaintsPolicy: - description: "NodeTaintsPolicy indicates how we will treat - node taints when calculating pod topology spread skew. - Options are: - Honor: nodes without taints, along with - tainted nodes for which the incoming pod has a toleration, - are included. - Ignore: node taints are ignored. All nodes - are included. \n If this value is nil, the behavior is - equivalent to the Ignore policy. This is a beta-level - feature default enabled by the NodeInclusionPolicyInPodTopologySpread - feature flag." + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string topologyKey: - description: TopologyKey is the key of node labels. Nodes - that have a label with this key and identical values are - considered to be in the same topology. We consider each - as a "bucket", and try to put balanced number - of pods into each bucket. We define a domain as a particular - instance of a topology. Also, we define an eligible domain - as a domain whose nodes meet the requirements of nodeAffinityPolicy - and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", - each Node is a domain of that topology. And, if TopologyKey - is "topology.kubernetes.io/zone", each zone is a domain - of that topology. It's a required field. + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. type: string whenUnsatisfiable: - description: 'WhenUnsatisfiable indicates how to deal with - a pod if it doesn''t satisfy the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule - it. - ScheduleAnyway tells the scheduler to schedule the - pod in any location, but giving higher precedence to topologies - that would help reduce the skew. A constraint is considered - "Unsatisfiable" for an incoming pod if and only if every - possible node assignment for that pod would violate "MaxSkew" - on some topology. For example, in a 3-zone cluster, MaxSkew - is set to 1, and pods with the same labelSelector spread - as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming - pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) - as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). - In other words, the cluster can still be imbalanced, but - scheduler won''t make it *more* imbalanced. It''s a required - field.' + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. type: string required: - maxSkew @@ -1634,32 +1605,33 @@ spec: type: array type: object selector: - description: 'selector is a label query over pods that should match - the replica count. It must match the pod template''s labels. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + description: |- + selector is a label query over pods that should match the replica count. + It must match the pod template's labels. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1672,11 +1644,10 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic @@ -1706,8 +1677,9 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true storageClass: - description: 'StorageClass defines the name of the StorageClass required - by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: |- + StorageClass defines the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 type: string volumeClaimTemplate: description: VolumeClaimTemplate defines the volume claim template @@ -1724,9 +1696,9 @@ spec: description: EtcdStatus defines the observed state of Etcd. properties: clusterSize: - description: 'Cluster size is the current size of the etcd cluster. - Deprecated: this field will not be populated with any value and - will be removed in the future.' + description: |- + Cluster size is the current size of the etcd cluster. + Deprecated: this field will not be populated with any value and will be removed in the future. format: int32 type: integer conditions: @@ -1787,32 +1759,33 @@ spec: type: string type: object labelSelector: - description: 'LabelSelector is a label query over pods that should - match the replica count. It must match the pod template''s labels. - Deprecated: this field will be removed in the future.' + description: |- + LabelSelector is a label query over pods that should match the replica count. + It must match the pod template's labels. + Deprecated: this field will be removed in the future. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1825,17 +1798,17 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic lastError: - description: 'LastError represents the last occurred error. Deprecated: - Use LastErrors instead.' + description: |- + LastError represents the last occurred error. + Deprecated: Use LastErrors instead. type: string lastErrors: description: LastErrors captures errors that occurred during the last @@ -1852,14 +1825,14 @@ spec: description: Description is a human-readable message indicating details of the error. type: string - lastUpdateTime: - description: LastUpdateTime is the time the error was reported. + observedAt: + description: ObservedAt is the time the error was observed. format: date-time type: string required: - code - description - - lastUpdateTime + - observedAt type: object type: array lastOperation: @@ -1874,15 +1847,13 @@ spec: was updated. format: date-time type: string - rundID: - description: RunID correlates an operation with a reconciliation - run. Every time an etcd resource is reconciled (barring status - reconciliation which is periodic), a unique ID is generated - which can be used to correlate all actions done as part of a - single reconcile run. Capturing this as part of LastOperation - aids in establishing this correlation. This further helps in - also easily filtering reconcile logs as all structured logs - in a reconcile run should have the `runID` referenced. + runID: + description: |- + RunID correlates an operation with a reconciliation run. + Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is + generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this + as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering + reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. type: string state: description: State is the state of the last operation. @@ -1893,7 +1864,7 @@ spec: required: - description - lastUpdateTime - - rundID + - runID - state - type type: object @@ -1954,13 +1925,14 @@ spec: format: int32 type: integer serviceName: - description: 'ServiceName is the name of the etcd service. Deprecated: - this field will be removed in the future.' + description: |- + ServiceName is the name of the etcd service. + Deprecated: this field will be removed in the future. type: string updatedReplicas: - description: 'UpdatedReplicas is the count of updated replicas in - the etcd cluster. Deprecated: this field will be removed in the - future.' + description: |- + UpdatedReplicas is the count of updated replicas in the etcd cluster. + Deprecated: this field will be removed in the future. format: int32 type: integer type: object diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 16951ef91..9aa0b085e 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -25,14 +25,19 @@ spec: source to a target store. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -41,9 +46,9 @@ spec: backups task. properties: maxBackupAge: - description: MaxBackupAge is the maximum age in days that a backup - must have in order to be copied. By default all backups will be - copied. + description: |- + MaxBackupAge is the maximum age in days that a backup must have in order to be copied. + By default all backups will be copied. format: int32 type: integer maxBackups: @@ -122,8 +127,8 @@ spec: snapshot before copying backups. type: boolean timeout: - description: Timeout is the timeout for waiting for a final full - snapshot. When this timeout expires, the copying of backups + description: |- + Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups will be performed anyway. No timeout or 0 means wait forever. type: string required: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index cdbce78a8..cfccfd992 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -48,14 +48,19 @@ spec: description: Etcd is the Schema for the etcds API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -71,23 +76,29 @@ spec: and delta snapshots of etcd. properties: compactionResources: - description: 'CompactionResources defines compute Resources required - by compaction job. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: |- + CompactionResources defines compute Resources required by compaction job. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ properties: claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string required: - name @@ -103,8 +114,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -113,11 +125,19 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true +<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' +======= + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +>>>>>>> 8571402f (Fix and run `make generate`) type: object type: object compression: @@ -148,10 +168,9 @@ spec: delta snapshots will be taken type: string deltaSnapshotRetentionPeriod: - description: DeltaSnapshotRetentionPeriod defines the duration - for which delta snapshots will be retained, excluding the latest - snapshot set. The value should be a string formatted as a duration - (e.g., '1s', '2m', '3h', '4d') + description: |- + DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set. + The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d') pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ type: string enableProfiling: @@ -205,23 +224,29 @@ spec: format: int32 type: integer resources: - description: 'Resources defines compute Resources required by - backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: |- + Resources defines compute Resources required by backup-restore container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ properties: claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string required: - name @@ -237,8 +262,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -247,11 +273,19 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true +<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' +======= + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +>>>>>>> 8571402f (Fix and run `make generate`) type: object type: object store: @@ -289,8 +323,9 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -303,8 +338,9 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -342,8 +378,9 @@ spec: description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: - description: SecretReference represents a Secret Reference. It - has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -380,8 +417,9 @@ spec: TLS secrets for client communication to ETCD cluster properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -394,8 +432,9 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -451,14 +490,14 @@ spec: - extensive type: string peerUrlTls: - description: PeerUrlTLS contains the ca and server TLS secrets - for peer communication within ETCD cluster Currently, PeerUrlTLS - does not require client TLS secrets for gardener implementation - of ETCD cluster. + description: |- + PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster + Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -471,8 +510,9 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: description: name is unique within a namespace to reference @@ -513,23 +553,29 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resources: - description: 'Resources defines the compute Resources required - by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: |- + Resources defines the compute Resources required by etcd container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ properties: claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string required: - name @@ -545,8 +591,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -555,11 +602,19 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true +<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' +======= + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +>>>>>>> 8571402f (Fix and run `make generate`) type: object type: object serverPort: @@ -578,35 +633,35 @@ spec: format: int32 type: integer schedulingConstraints: - description: SchedulingConstraints defines the different scheduling - constraints that must be applied to the pod spec in the etcd statefulset. + description: |- + SchedulingConstraints defines the different scheduling constraints that must be applied to the + pod spec in the etcd statefulset. Currently supported constraints are Affinity and TopologySpreadConstraints. properties: affinity: - description: Affinity defines the various affinity and anti-affinity - rules for a pod that are honoured by the kube-scheduler. + description: |- + Affinity defines the various affinity and anti-affinity rules for a pod + that are honoured by the kube-scheduler. properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: description: A node selector term, associated with @@ -616,32 +671,26 @@ spec: description: A list of node selector requirements by node's labels. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -654,32 +703,26 @@ spec: description: A list of node selector requirements by node's fields. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -701,53 +744,46 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -760,32 +796,26 @@ spec: description: A list of node selector requirements by node's fields. items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -808,18 +838,16 @@ spec: other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred @@ -838,30 +866,25 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -873,53 +896,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by - this field and the ones listed in the namespaces - field. null selector and null or empty namespaces - list means "this pod's namespace". An empty - selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -931,42 +946,37 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. - The term is applied to the union of the namespaces - listed in this field and the ones selected - by namespaceSelector. null or empty namespaces - list and null namespaceSelector means "this - pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -975,23 +985,22 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, @@ -1002,28 +1011,24 @@ spec: selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1036,51 +1041,44 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by this - field and the ones listed in the namespaces field. - null selector and null or empty namespaces list - means "this pod's namespace". An empty selector - ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1093,33 +1091,29 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. The - term is applied to the union of the namespaces - listed in this field and the ones selected by - namespaceSelector. null or empty namespaces list - and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -1132,18 +1126,16 @@ spec: as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred @@ -1162,30 +1154,25 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -1197,53 +1184,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by - this field and the ones listed in the namespaces - field. null selector and null or empty namespaces - list means "this pod's namespace". An empty - selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -1255,42 +1234,37 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. - The term is applied to the union of the namespaces - listed in this field and the ones selected - by namespaceSelector. null or empty namespaces - list and null namespaceSelector means "this - pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -1299,23 +1273,22 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, @@ -1326,28 +1299,24 @@ spec: selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1360,51 +1329,44 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: A label query over the set of namespaces - that the term applies to. The term is applied - to the union of the namespaces selected by this - field and the ones listed in the namespaces field. - null selector and null or empty namespaces list - means "this pod's namespace". An empty selector - ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1417,33 +1379,29 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list - of namespace names that the term applies to. The - term is applied to the union of the namespaces - listed in this field and the ones selected by - namespaceSelector. null or empty namespaces list - and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -1452,24 +1410,25 @@ spec: type: object type: object topologySpreadConstraints: - description: TopologySpreadConstraints describes how a group of - pods ought to spread across topology domains, that are honoured - by the kube-scheduler. + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology domains, + that are honoured by the kube-scheduler. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: - description: LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine - the number of pods in their corresponding topology domain. + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: @@ -1477,17 +1436,16 @@ spec: applies to. type: string operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, - NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists - or DoesNotExist, the values array must be empty. - This array is replaced during a strategic merge - patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -1499,15 +1457,15 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field - is "key", the operator is "In", and the values array - contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: +<<<<<<< HEAD description: "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod @@ -1521,110 +1479,123 @@ spec: against labelSelector. \n This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default)." +======= + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. +>>>>>>> 8571402f (Fix and run `make generate`) items: type: string type: array x-kubernetes-list-type: atomic maxSkew: - description: 'MaxSkew describes the degree to which pods - may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, - it is the maximum permitted difference between the number - of matching pods in the target topology and the global - minimum. The global minimum is the minimum number of matching - pods in an eligible domain or zero if the number of eligible - domains is less than MinDomains. For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with the same labelSelector - spread as 2/2/1: In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to - zone3 to become 2/2/2; scheduling it onto zone1(zone2) - would make the ActualSkew(3-1) on zone1(zone2) violate - MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled - onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, - it is used to give higher precedence to topologies that - satisfy it. It''s a required field. Default value is 1 - and 0 is not allowed.' + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. format: int32 type: integer minDomains: - description: "MinDomains indicates a minimum number of eligible - domains. When the number of eligible domains with matching - topology keys is less than minDomains, Pod Topology Spread - treats \"global minimum\" as 0, and then the calculation - of Skew is performed. And when the number of eligible - domains with matching topology keys equals or greater - than minDomains, this value has no effect on scheduling. - As a result, when the number of eligible domains is less - than minDomains, scheduler won't schedule more than maxSkew - Pods to those domains. If value is nil, the constraint - behaves as if MinDomains is equal to 1. Valid values are - integers greater than 0. When value is not nil, WhenUnsatisfiable - must be DoNotSchedule. \n For example, in a 3-zone cluster, - MaxSkew is set to 2, MinDomains is set to 5 and pods with - the same labelSelector spread as 2/2/2: | zone1 | zone2 - | zone3 | | P P | P P | P P | The number of domains - is less than 5(MinDomains), so \"global minimum\" is treated - as 0. In this situation, new pod with the same labelSelector - cannot be scheduled, because computed skew will be 3(3 - - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. \n This is a beta field and requires - the MinDomainsInPodTopologySpread feature gate to be enabled - (enabled by default)." + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). format: int32 type: integer nodeAffinityPolicy: - description: "NodeAffinityPolicy indicates how we will treat - Pod's nodeAffinity/nodeSelector when calculating pod topology - spread skew. Options are: - Honor: only nodes matching - nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes - are included in the calculations. \n If this value is - nil, the behavior is equivalent to the Honor policy. This - is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread - feature flag." + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string nodeTaintsPolicy: - description: "NodeTaintsPolicy indicates how we will treat - node taints when calculating pod topology spread skew. - Options are: - Honor: nodes without taints, along with - tainted nodes for which the incoming pod has a toleration, - are included. - Ignore: node taints are ignored. All nodes - are included. \n If this value is nil, the behavior is - equivalent to the Ignore policy. This is a beta-level - feature default enabled by the NodeInclusionPolicyInPodTopologySpread - feature flag." + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string topologyKey: - description: TopologyKey is the key of node labels. Nodes - that have a label with this key and identical values are - considered to be in the same topology. We consider each - as a "bucket", and try to put balanced number - of pods into each bucket. We define a domain as a particular - instance of a topology. Also, we define an eligible domain - as a domain whose nodes meet the requirements of nodeAffinityPolicy - and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", - each Node is a domain of that topology. And, if TopologyKey - is "topology.kubernetes.io/zone", each zone is a domain - of that topology. It's a required field. + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. type: string whenUnsatisfiable: - description: 'WhenUnsatisfiable indicates how to deal with - a pod if it doesn''t satisfy the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule - it. - ScheduleAnyway tells the scheduler to schedule the - pod in any location, but giving higher precedence to topologies - that would help reduce the skew. A constraint is considered - "Unsatisfiable" for an incoming pod if and only if every - possible node assignment for that pod would violate "MaxSkew" - on some topology. For example, in a 3-zone cluster, MaxSkew - is set to 1, and pods with the same labelSelector spread - as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming - pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) - as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). - In other words, the cluster can still be imbalanced, but - scheduler won''t make it *more* imbalanced. It''s a required - field.' + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. type: string required: - maxSkew @@ -1634,32 +1605,33 @@ spec: type: array type: object selector: - description: 'selector is a label query over pods that should match - the replica count. It must match the pod template''s labels. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + description: |- + selector is a label query over pods that should match the replica count. + It must match the pod template's labels. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1672,11 +1644,10 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic @@ -1706,8 +1677,9 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true storageClass: - description: 'StorageClass defines the name of the StorageClass required - by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: |- + StorageClass defines the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 type: string volumeClaimTemplate: description: VolumeClaimTemplate defines the volume claim template @@ -1724,9 +1696,9 @@ spec: description: EtcdStatus defines the observed state of Etcd. properties: clusterSize: - description: 'Cluster size is the current size of the etcd cluster. - Deprecated: this field will not be populated with any value and - will be removed in the future.' + description: |- + Cluster size is the current size of the etcd cluster. + Deprecated: this field will not be populated with any value and will be removed in the future. format: int32 type: integer conditions: @@ -1787,32 +1759,33 @@ spec: type: string type: object labelSelector: - description: 'LabelSelector is a label query over pods that should - match the replica count. It must match the pod template''s labels. - Deprecated: this field will be removed in the future.' + description: |- + LabelSelector is a label query over pods that should match the replica count. + It must match the pod template's labels. + Deprecated: this field will be removed in the future. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1825,17 +1798,17 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic lastError: - description: 'LastError represents the last occurred error. Deprecated: - Use LastErrors instead.' + description: |- + LastError represents the last occurred error. + Deprecated: Use LastErrors instead. type: string lastErrors: description: LastErrors captures errors that occurred during the last @@ -1852,14 +1825,14 @@ spec: description: Description is a human-readable message indicating details of the error. type: string - lastUpdateTime: - description: LastUpdateTime is the time the error was reported. + observedAt: + description: ObservedAt is the time the error was observed. format: date-time type: string required: - code - description - - lastUpdateTime + - observedAt type: object type: array lastOperation: @@ -1874,15 +1847,13 @@ spec: was updated. format: date-time type: string - rundID: - description: RunID correlates an operation with a reconciliation - run. Every time an etcd resource is reconciled (barring status - reconciliation which is periodic), a unique ID is generated - which can be used to correlate all actions done as part of a - single reconcile run. Capturing this as part of LastOperation - aids in establishing this correlation. This further helps in - also easily filtering reconcile logs as all structured logs - in a reconcile run should have the `runID` referenced. + runID: + description: |- + RunID correlates an operation with a reconciliation run. + Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is + generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this + as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering + reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. type: string state: description: State is the state of the last operation. @@ -1893,7 +1864,7 @@ spec: required: - description - lastUpdateTime - - rundID + - runID - state - type type: object @@ -1954,13 +1925,14 @@ spec: format: int32 type: integer serviceName: - description: 'ServiceName is the name of the etcd service. Deprecated: - this field will be removed in the future.' + description: |- + ServiceName is the name of the etcd service. + Deprecated: this field will be removed in the future. type: string updatedReplicas: - description: 'UpdatedReplicas is the count of updated replicas in - the etcd cluster. Deprecated: this field will be removed in the - future.' + description: |- + UpdatedReplicas is the count of updated replicas in the etcd cluster. + Deprecated: this field will be removed in the future. format: int32 type: integer type: object diff --git a/hack/tools.mk b/hack/tools.mk index a99394715..438c2c7c2 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -18,4 +18,4 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) ######################################### $(KUSTOMIZE): - @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v4@${KUSTOMIZE_VERSION} + @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION} \ No newline at end of file From 8c5ccaea7d05e868929b299d2ca17eb9f38e8c0e Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 14 Mar 2024 14:04:20 +0530 Subject: [PATCH 098/235] Pull changes from ce2f556bf73d0e1e483527586cf267dc1437b1f0 --- internal/features/features.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/features/features.go b/internal/features/features.go index a7941de36..30a96cb76 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -31,11 +31,12 @@ const ( // changes required for the usage of the etcd-wrapper image. // owner @unmarshall @aaronfern // alpha: v0.19 + // beta: v0.22 UseEtcdWrapper featuregate.Feature = "UseEtcdWrapper" ) var defaultFeatures = map[featuregate.Feature]featuregate.FeatureSpec{ - UseEtcdWrapper: {Default: false, PreRelease: featuregate.Alpha}, + UseEtcdWrapper: {Default: true, PreRelease: featuregate.Beta}, } // GetDefaultFeatures returns the default feature gates known to etcd-druid. From 6d3c9f0763a362a34b918db08549254e64d9b029 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 14 Mar 2024 14:15:23 +0530 Subject: [PATCH 099/235] Pull changes from 5cd5b1bd8e63a8cd5abd9c93b8b1021c6ce47853 --- internal/operator/statefulset/builder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index e564076f4..0bc3c6042 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -28,6 +28,7 @@ const ( defaultServerPort int32 = 2380 defaultClientPort int32 = 2379 defaultWrapperPort int = 9095 + defaultMaxBackupsLimitBasedGC int32 = 7 defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi defaultHeartbeatDuration = "10s" @@ -513,7 +514,7 @@ func (b *stsBuilder) getBackupStoreCommandArgs() []string { } commandArgs = append(commandArgs, fmt.Sprintf("--garbage-collection-policy=%s", garbageCollectionPolicy)) if garbageCollectionPolicy == "LimitBased" { - commandArgs = append(commandArgs, "--max-backups=7") + commandArgs = append(commandArgs, fmt.Sprintf("--max-backups=%d", utils.TypeDeref[int32](b.etcd.Spec.Backup.MaxBackupsLimitBasedGC, defaultMaxBackupsLimitBasedGC))) } if b.etcd.Spec.Backup.GarbageCollectionPeriod != nil { commandArgs = append(commandArgs, fmt.Sprintf("--garbage-collection-period=%s", b.etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String())) From 2385d0ff185aec4907b2742cda0203d8e0aef02e Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 14 Mar 2024 16:15:31 +0530 Subject: [PATCH 100/235] Pull changes from 9c5f8254e3aeb24c1e3e88d17d8d1de336ce981b --- .github/dependabot.yml | 14 ++------------ api/v1alpha1/types_constant.go | 4 ++++ hack/addlicenseheaders.sh | 11 ++++++++++- hack/boilerplate.go.txt | 6 +++--- internal/client/kubernetes/types.go | 14 ++------------ internal/common/constants.go | 14 ++------------ internal/controller/compaction/config.go | 14 ++------------ internal/controller/compaction/metrics.go | 14 ++------------ internal/controller/compaction/reconciler.go | 14 ++------------ internal/controller/compaction/register.go | 14 ++------------ internal/controller/config.go | 14 ++------------ internal/controller/etcd/config.go | 4 ++++ internal/controller/etcd/reconcile_delete.go | 4 ++++ internal/controller/etcd/reconcile_spec.go | 14 ++------------ internal/controller/etcd/reconcile_status.go | 4 ++++ internal/controller/etcd/reconciler.go | 14 ++------------ internal/controller/etcd/register.go | 4 ++++ internal/controller/etcd/register_test.go | 4 ++++ .../controller/etcdcopybackupstask/config.go | 14 ++------------ .../etcdcopybackupstask_suite_test.go | 14 ++------------ .../controller/etcdcopybackupstask/reconciler.go | 14 ++------------ .../etcdcopybackupstask/reconciler_test.go | 14 ++------------ .../controller/etcdcopybackupstask/register.go | 14 ++------------ internal/controller/manager.go | 14 ++------------ internal/controller/predicate/predicate.go | 14 ++------------ .../controller/predicate/predicate_suite_test.go | 14 ++------------ internal/controller/predicate/predicate_test.go | 14 ++------------ internal/controller/secret/config.go | 14 ++------------ internal/controller/secret/reconciler.go | 14 ++------------ internal/controller/secret/reconciler_test.go | 14 ++------------ internal/controller/secret/register.go | 14 ++------------ internal/controller/secret/secret_suite_test.go | 14 ++------------ internal/controller/utils/etcdstatus.go | 4 ++++ internal/controller/utils/reconciler.go | 14 ++------------ internal/controller/utils/utils_suite_test.go | 14 ++------------ internal/controller/utils/validator.go | 14 ++------------ internal/controller/utils/validator_test.go | 14 ++------------ internal/errors/errors.go | 4 ++++ internal/features/features.go | 14 ++------------ internal/health/condition/builder.go | 14 ++------------ internal/health/condition/builder_test.go | 14 ++------------ internal/health/condition/check_all_members.go | 14 ++------------ .../health/condition/check_all_members_test.go | 14 ++------------ internal/health/condition/check_backup_ready.go | 14 ++------------ .../health/condition/check_backup_ready_test.go | 14 ++------------ .../health/condition/check_data_volumes_ready.go | 14 ++------------ internal/health/condition/check_ready.go | 14 ++------------ internal/health/condition/check_ready_test.go | 14 ++------------ .../health/condition/condition_suite_test.go | 14 ++------------ internal/health/condition/types.go | 14 ++------------ internal/health/etcdmember/builder.go | 14 ++------------ internal/health/etcdmember/builder_test.go | 14 ++------------ internal/health/etcdmember/check_ready.go | 14 ++------------ internal/health/etcdmember/check_ready_test.go | 14 ++------------ .../health/etcdmember/etcdmember_suite_test.go | 14 ++------------ internal/health/etcdmember/types.go | 14 ++------------ internal/health/status/check.go | 14 ++------------ internal/health/status/check_test.go | 14 ++------------ internal/health/status/status_suite_test.go | 14 ++------------ internal/mapper/etcd_to_secret.go | 14 ++------------ internal/mapper/etcd_to_secret_test.go | 14 ++------------ internal/mapper/mapper_suite_test.go | 14 ++------------ internal/mapper/statefulset_to_etcd.go | 14 ++------------ internal/mapper/statefulset_to_etcd_test.go | 14 ++------------ internal/metrics/metrics.go | 14 ++------------ internal/metrics/metrics_suite_test.go | 14 ++------------ internal/metrics/metrics_test.go | 14 ++------------ internal/mock/controller-runtime/client/doc.go | 16 +++------------- internal/mock/controller-runtime/manager/doc.go | 4 ++++ internal/operator/clientservice/clientservice.go | 4 ++++ .../operator/clientservice/clientservice_test.go | 14 ++------------ internal/operator/component/types.go | 4 ++++ internal/operator/configmap/configmap.go | 4 ++++ internal/operator/configmap/configmap_test.go | 4 ++++ internal/operator/configmap/etcdconfig.go | 4 ++++ internal/operator/memberlease/memberlease.go | 4 ++++ .../operator/memberlease/memberlease_test.go | 4 ++++ internal/operator/peerservice/peerservice.go | 4 ++++ .../operator/peerservice/peerservice_test.go | 14 ++------------ .../poddistruptionbudget/poddisruptionbudget.go | 4 ++++ .../poddisruptionbudget_test.go | 4 ++++ internal/operator/registry.go | 4 ++++ internal/operator/role/role.go | 4 ++++ internal/operator/role/role_test.go | 4 ++++ internal/operator/rolebinding/rolebinding.go | 4 ++++ .../operator/rolebinding/rolebinding_test.go | 4 ++++ .../operator/serviceaccount/serviceaccount.go | 4 ++++ .../serviceaccount/serviceaccount_test.go | 4 ++++ internal/operator/snapshotlease/snapshotlease.go | 4 ++++ .../operator/snapshotlease/snapshotlease_test.go | 4 ++++ internal/operator/statefulset/builder.go | 4 ++++ internal/operator/statefulset/constants.go | 4 ++++ internal/operator/statefulset/statefulset.go | 4 ++++ .../operator/statefulset/statefulset_test.go | 4 ++++ internal/operator/statefulset/stsmatcher.go | 4 ++++ internal/utils/concurrent.go | 4 ++++ internal/utils/envvar.go | 4 ++++ internal/utils/image.go | 14 ++------------ internal/utils/image_test.go | 4 ++++ internal/utils/lease.go | 14 ++------------ internal/utils/lease_test.go | 4 ++++ internal/utils/miscellaneous.go | 14 ++------------ internal/utils/statefulset.go | 14 ++------------ internal/utils/statefulset_test.go | 4 ++++ internal/utils/store.go | 14 ++------------ internal/utils/store_test.go | 4 ++++ internal/utils/utils_suite_test.go | 14 ++------------ internal/version/version.go | 14 ++------------ internal/webhook/sentinel/config.go | 4 ++++ internal/webhook/sentinel/handler.go | 4 ++++ internal/webhook/sentinel/register.go | 4 ++++ test/it/controller/assets/assets.go | 14 ++------------ test/it/controller/etcd/assertions.go | 4 ++++ test/it/controller/etcd/reconciler_test.go | 4 ++++ test/it/setup/setup.go | 4 ++++ test/utils/client.go | 4 ++++ test/utils/constants.go | 4 ++++ test/utils/errors.go | 4 ++++ test/utils/imagevector.go | 4 ++++ test/utils/matcher.go | 4 ++++ 120 files changed, 350 insertions(+), 821 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6ca7fcd4c..e5d42e3d5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,16 +1,6 @@ -# Copyright 2023 SAP SE or an SAP affiliate company +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/types_constant.go index 1491fdf1d..6c3d37d41 100644 --- a/api/v1alpha1/types_constant.go +++ b/api/v1alpha1/types_constant.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package v1alpha1 // Common label keys to be placed on all druid managed resources diff --git a/hack/addlicenseheaders.sh b/hack/addlicenseheaders.sh index bed201612..f753069c2 100755 --- a/hack/addlicenseheaders.sh +++ b/hack/addlicenseheaders.sh @@ -8,13 +8,22 @@ set -e echo "> Adding Apache License header to all go files where it is not present" +YEAR=$1 +if [[ -z "$1" ]]; then + cat << EOF +Unspecified 'YEAR' argument. +Usage: addlicenceheaders.sh +EOF + exit 1 +fi + # addlicence with a license file (parameter -f) expects no comments in the file. # boilerplate.go.txt is however also used also when generating go code. # Therefore we remove '//' from boilerplate.go.txt here before passing it to addlicense. temp_file=$(mktemp) trap "rm -f $temp_file" EXIT -sed 's|^// *||' hack/boilerplate.go.txt > $temp_file +sed "s/{YEAR}/${YEAR}/g" hack/boilerplate.go.txt > $temp_file addlicense \ -f $temp_file \ diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 3f66bff89..9266b594b 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,3 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 \ No newline at end of file +SPDX-FileCopyrightText: {YEAR} SAP SE or an SAP affiliate company and Gardener contributors + +SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/internal/client/kubernetes/types.go b/internal/client/kubernetes/types.go index 3fe5155ec..12237375d 100644 --- a/internal/client/kubernetes/types.go +++ b/internal/client/kubernetes/types.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package kubernetes diff --git a/internal/common/constants.go b/internal/common/constants.go index 455fa825b..89f464360 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -1,16 +1,6 @@ -// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package common diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index af46ec88b..4d82477d5 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package compaction diff --git a/internal/controller/compaction/metrics.go b/internal/controller/compaction/metrics.go index dac70089b..243fd00a0 100644 --- a/internal/controller/compaction/metrics.go +++ b/internal/controller/compaction/metrics.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package compaction diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 7f0d75983..846fdfb2a 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package compaction diff --git a/internal/controller/compaction/register.go b/internal/controller/compaction/register.go index 92165c568..f8dd65db6 100644 --- a/internal/controller/compaction/register.go +++ b/internal/controller/compaction/register.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package compaction diff --git a/internal/controller/config.go b/internal/controller/config.go index 8a3f72322..e4d6213d7 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package controller diff --git a/internal/controller/etcd/config.go b/internal/controller/etcd/config.go index 96eb9e3c9..307da8e97 100644 --- a/internal/controller/etcd/config.go +++ b/internal/controller/etcd/config.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index b7d4d4900..e179ba6b7 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index c26fa2872..3f2c4217e 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcd diff --git a/internal/controller/etcd/reconcile_status.go b/internal/controller/etcd/reconcile_status.go index 9ead22979..afaefe32b 100644 --- a/internal/controller/etcd/reconcile_status.go +++ b/internal/controller/etcd/reconcile_status.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 70ea552be..be2fc1e9b 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcd diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 94d6e2d00..2e1a1c5ff 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/internal/controller/etcd/register_test.go b/internal/controller/etcd/register_test.go index a9fe3cd96..618f3e10e 100644 --- a/internal/controller/etcd/register_test.go +++ b/internal/controller/etcd/register_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/internal/controller/etcdcopybackupstask/config.go b/internal/controller/etcdcopybackupstask/config.go index 0290b7417..9bde160db 100644 --- a/internal/controller/etcdcopybackupstask/config.go +++ b/internal/controller/etcdcopybackupstask/config.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdcopybackupstask diff --git a/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go b/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go index 5ebe73b1a..52515953e 100644 --- a/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go +++ b/internal/controller/etcdcopybackupstask/etcdcopybackupstask_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdcopybackupstask diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 0a5628ec1..1f21546c3 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdcopybackupstask diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 20d5bed28..59c300321 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdcopybackupstask diff --git a/internal/controller/etcdcopybackupstask/register.go b/internal/controller/etcdcopybackupstask/register.go index 30307ad12..7eb8172be 100644 --- a/internal/controller/etcdcopybackupstask/register.go +++ b/internal/controller/etcdcopybackupstask/register.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdcopybackupstask diff --git a/internal/controller/manager.go b/internal/controller/manager.go index e20d42282..d9c77b6a2 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package controller diff --git a/internal/controller/predicate/predicate.go b/internal/controller/predicate/predicate.go index ba588f1b3..58554ddbc 100644 --- a/internal/controller/predicate/predicate.go +++ b/internal/controller/predicate/predicate.go @@ -1,16 +1,6 @@ -// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package predicate diff --git a/internal/controller/predicate/predicate_suite_test.go b/internal/controller/predicate/predicate_suite_test.go index 0829a0f8e..d800fe30e 100644 --- a/internal/controller/predicate/predicate_suite_test.go +++ b/internal/controller/predicate/predicate_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package predicate_test diff --git a/internal/controller/predicate/predicate_test.go b/internal/controller/predicate/predicate_test.go index e4cf87fe0..f3fc75231 100644 --- a/internal/controller/predicate/predicate_test.go +++ b/internal/controller/predicate/predicate_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package predicate_test diff --git a/internal/controller/secret/config.go b/internal/controller/secret/config.go index db458dacd..ca9e207fd 100644 --- a/internal/controller/secret/config.go +++ b/internal/controller/secret/config.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package secret diff --git a/internal/controller/secret/reconciler.go b/internal/controller/secret/reconciler.go index d60e7b627..454479223 100644 --- a/internal/controller/secret/reconciler.go +++ b/internal/controller/secret/reconciler.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package secret diff --git a/internal/controller/secret/reconciler_test.go b/internal/controller/secret/reconciler_test.go index 9a87bedfa..c2d45be7d 100644 --- a/internal/controller/secret/reconciler_test.go +++ b/internal/controller/secret/reconciler_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package secret diff --git a/internal/controller/secret/register.go b/internal/controller/secret/register.go index 3c5fa8614..44ab3824c 100644 --- a/internal/controller/secret/register.go +++ b/internal/controller/secret/register.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package secret diff --git a/internal/controller/secret/secret_suite_test.go b/internal/controller/secret/secret_suite_test.go index a3598ae8f..8286b0674 100644 --- a/internal/controller/secret/secret_suite_test.go +++ b/internal/controller/secret/secret_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package secret diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index 096b42394..e67935328 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 059a4989b..9185c04af 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/controller/utils/utils_suite_test.go b/internal/controller/utils/utils_suite_test.go index f825cd8e7..a3123f5d4 100644 --- a/internal/controller/utils/utils_suite_test.go +++ b/internal/controller/utils/utils_suite_test.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/controller/utils/validator.go b/internal/controller/utils/validator.go index d3ec4d324..a189de47d 100644 --- a/internal/controller/utils/validator.go +++ b/internal/controller/utils/validator.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/controller/utils/validator_test.go b/internal/controller/utils/validator_test.go index db1f63d30..64719fc5d 100644 --- a/internal/controller/utils/validator_test.go +++ b/internal/controller/utils/validator_test.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 781b1b858..e519315b6 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package errors import ( diff --git a/internal/features/features.go b/internal/features/features.go index 30a96cb76..0bd7c799b 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package features diff --git a/internal/health/condition/builder.go b/internal/health/condition/builder.go index ca796c867..f979112af 100644 --- a/internal/health/condition/builder.go +++ b/internal/health/condition/builder.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition diff --git a/internal/health/condition/builder_test.go b/internal/health/condition/builder_test.go index d9f868c5a..17f5a6da0 100644 --- a/internal/health/condition/builder_test.go +++ b/internal/health/condition/builder_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition_test diff --git a/internal/health/condition/check_all_members.go b/internal/health/condition/check_all_members.go index d0863b7d5..0b72c9f7a 100644 --- a/internal/health/condition/check_all_members.go +++ b/internal/health/condition/check_all_members.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition diff --git a/internal/health/condition/check_all_members_test.go b/internal/health/condition/check_all_members_test.go index 4187ac387..3fec05b25 100644 --- a/internal/health/condition/check_all_members_test.go +++ b/internal/health/condition/check_all_members_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition_test diff --git a/internal/health/condition/check_backup_ready.go b/internal/health/condition/check_backup_ready.go index 4551faa9a..6d1fd3900 100644 --- a/internal/health/condition/check_backup_ready.go +++ b/internal/health/condition/check_backup_ready.go @@ -1,16 +1,6 @@ -// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition diff --git a/internal/health/condition/check_backup_ready_test.go b/internal/health/condition/check_backup_ready_test.go index 064bdeb02..4216d4361 100644 --- a/internal/health/condition/check_backup_ready_test.go +++ b/internal/health/condition/check_backup_ready_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition_test diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go index 18cb2fcd9..b2ba8c76d 100644 --- a/internal/health/condition/check_data_volumes_ready.go +++ b/internal/health/condition/check_data_volumes_ready.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition diff --git a/internal/health/condition/check_ready.go b/internal/health/condition/check_ready.go index e0ef3d1b2..50bd74e9b 100644 --- a/internal/health/condition/check_ready.go +++ b/internal/health/condition/check_ready.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition diff --git a/internal/health/condition/check_ready_test.go b/internal/health/condition/check_ready_test.go index 928e9972f..bb7235671 100644 --- a/internal/health/condition/check_ready_test.go +++ b/internal/health/condition/check_ready_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition_test diff --git a/internal/health/condition/condition_suite_test.go b/internal/health/condition/condition_suite_test.go index ff4383fdb..7eb295e3e 100644 --- a/internal/health/condition/condition_suite_test.go +++ b/internal/health/condition/condition_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition_test diff --git a/internal/health/condition/types.go b/internal/health/condition/types.go index 98c3a3b7a..557327068 100644 --- a/internal/health/condition/types.go +++ b/internal/health/condition/types.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package condition diff --git a/internal/health/etcdmember/builder.go b/internal/health/etcdmember/builder.go index fa9a77052..eb643f2a2 100644 --- a/internal/health/etcdmember/builder.go +++ b/internal/health/etcdmember/builder.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdmember diff --git a/internal/health/etcdmember/builder_test.go b/internal/health/etcdmember/builder_test.go index 6881fab81..f5f570e1a 100644 --- a/internal/health/etcdmember/builder_test.go +++ b/internal/health/etcdmember/builder_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdmember_test diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index c0c6069eb..be3c03fcd 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdmember diff --git a/internal/health/etcdmember/check_ready_test.go b/internal/health/etcdmember/check_ready_test.go index 250c341d4..905ef89b5 100644 --- a/internal/health/etcdmember/check_ready_test.go +++ b/internal/health/etcdmember/check_ready_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdmember_test diff --git a/internal/health/etcdmember/etcdmember_suite_test.go b/internal/health/etcdmember/etcdmember_suite_test.go index ebba7f55a..9be90ecf9 100644 --- a/internal/health/etcdmember/etcdmember_suite_test.go +++ b/internal/health/etcdmember/etcdmember_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdmember_test diff --git a/internal/health/etcdmember/types.go b/internal/health/etcdmember/types.go index 315e2aeb1..7731bedd1 100644 --- a/internal/health/etcdmember/types.go +++ b/internal/health/etcdmember/types.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package etcdmember diff --git a/internal/health/status/check.go b/internal/health/status/check.go index 66ee834b5..242090374 100644 --- a/internal/health/status/check.go +++ b/internal/health/status/check.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package status diff --git a/internal/health/status/check_test.go b/internal/health/status/check_test.go index bba6944db..94abe91b8 100644 --- a/internal/health/status/check_test.go +++ b/internal/health/status/check_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package status_test diff --git a/internal/health/status/status_suite_test.go b/internal/health/status/status_suite_test.go index 79ac82d0e..290e28fa2 100644 --- a/internal/health/status/status_suite_test.go +++ b/internal/health/status/status_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package status_test diff --git a/internal/mapper/etcd_to_secret.go b/internal/mapper/etcd_to_secret.go index b705d2fda..4bf797dd7 100644 --- a/internal/mapper/etcd_to_secret.go +++ b/internal/mapper/etcd_to_secret.go @@ -1,16 +1,6 @@ -// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package mapper diff --git a/internal/mapper/etcd_to_secret_test.go b/internal/mapper/etcd_to_secret_test.go index 63a8ec977..ed7526817 100644 --- a/internal/mapper/etcd_to_secret_test.go +++ b/internal/mapper/etcd_to_secret_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package mapper_test diff --git a/internal/mapper/mapper_suite_test.go b/internal/mapper/mapper_suite_test.go index 4cf5eed2c..ed8100265 100644 --- a/internal/mapper/mapper_suite_test.go +++ b/internal/mapper/mapper_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package mapper_test diff --git a/internal/mapper/statefulset_to_etcd.go b/internal/mapper/statefulset_to_etcd.go index 0a3c81713..a174a39f7 100644 --- a/internal/mapper/statefulset_to_etcd.go +++ b/internal/mapper/statefulset_to_etcd.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package mapper diff --git a/internal/mapper/statefulset_to_etcd_test.go b/internal/mapper/statefulset_to_etcd_test.go index 6c6ce84ce..f3bff090e 100644 --- a/internal/mapper/statefulset_to_etcd_test.go +++ b/internal/mapper/statefulset_to_etcd_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package mapper_test diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 3c02c5b4f..762ee8c00 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package metrics diff --git a/internal/metrics/metrics_suite_test.go b/internal/metrics/metrics_suite_test.go index 506884539..8bf8a48cd 100644 --- a/internal/metrics/metrics_suite_test.go +++ b/internal/metrics/metrics_suite_test.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package metrics diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 1e8de70d8..7250cc7c8 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package metrics diff --git a/internal/mock/controller-runtime/client/doc.go b/internal/mock/controller-runtime/client/doc.go index fa2f38c39..cf1678271 100644 --- a/internal/mock/controller-runtime/client/doc.go +++ b/internal/mock/controller-runtime/client/doc.go @@ -1,16 +1,6 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//go:generate mockgen -package client -destination=mocks.go sigs.k8s.io/controller-runtime/pkg/client Client,StatusWriter,Reader,Writer +// SPDX-License-Identifier: Apache-2.0 + package client diff --git a/internal/mock/controller-runtime/manager/doc.go b/internal/mock/controller-runtime/manager/doc.go index d34952302..691185d01 100644 --- a/internal/mock/controller-runtime/manager/doc.go +++ b/internal/mock/controller-runtime/manager/doc.go @@ -1,2 +1,6 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + //go:generate mockgen -package manager -destination=mocks.go sigs.k8s.io/controller-runtime/pkg/manager Manager package manager diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 3b44c8490..8cc6c8ae4 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package clientservice import ( diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 6068bf3bd..c3c3efd47 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package clientservice diff --git a/internal/operator/component/types.go b/internal/operator/component/types.go index 7563b2ec5..3d4a560ec 100644 --- a/internal/operator/component/types.go +++ b/internal/operator/component/types.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package component import ( diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 37c96fb06..5fa199c0d 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package configmap import ( diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 8cee3d8ef..797f90e52 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package configmap import ( diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index 98fadda01..cea8cf404 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package configmap import ( diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 592b42ae9..c12f3408d 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package memberlease import ( diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index eadbd8b76..aa1ba8915 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package memberlease import ( diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 909cbc332..9fd82d215 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package peerservice import ( diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 3fce059df..86142c294 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package peerservice diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 29a803d29..c7557623f 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package poddistruptionbudget import ( diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go index 1823082a8..8d8b67e26 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package poddistruptionbudget import ( diff --git a/internal/operator/registry.go b/internal/operator/registry.go index 18858595f..08c00b9e5 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package operator import ( diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 2365e44d6..431c57a3b 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package role import ( diff --git a/internal/operator/role/role_test.go b/internal/operator/role/role_test.go index 39fc6d02c..1a9ee1b2c 100644 --- a/internal/operator/role/role_test.go +++ b/internal/operator/role/role_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package role import ( diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index 3b0ea7c08..e843dcec9 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package rolebinding import ( diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/operator/rolebinding/rolebinding_test.go index bf29092ec..330c84837 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/operator/rolebinding/rolebinding_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package rolebinding import ( diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 91ca5fbf3..8d1532c41 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package serviceaccount import ( diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/operator/serviceaccount/serviceaccount_test.go index d22e67d5c..77ab0a472 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/operator/serviceaccount/serviceaccount_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package serviceaccount import ( diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 700ca5b80..149c0015e 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package snapshotlease import ( diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 4373cd490..4ad911abb 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package snapshotlease import ( diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 0bc3c6042..d8f3d1427 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package statefulset import ( diff --git a/internal/operator/statefulset/constants.go b/internal/operator/statefulset/constants.go index 8a96f1f15..f6940d133 100644 --- a/internal/operator/statefulset/constants.go +++ b/internal/operator/statefulset/constants.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package statefulset // constants for volume names diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 6e886ccf5..208573b38 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package statefulset import ( diff --git a/internal/operator/statefulset/statefulset_test.go b/internal/operator/statefulset/statefulset_test.go index d5eebea56..62845c91d 100644 --- a/internal/operator/statefulset/statefulset_test.go +++ b/internal/operator/statefulset/statefulset_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package statefulset import ( diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 366a8271a..efa8dded6 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package statefulset import ( diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go index 66c44645d..079423a40 100644 --- a/internal/utils/concurrent.go +++ b/internal/utils/concurrent.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index 5342b030e..dde1f2aee 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/image.go b/internal/utils/image.go index 01e604be5..18f1cfc81 100755 --- a/internal/utils/image.go +++ b/internal/utils/image.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index 8a9b29630..ad4542c1b 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/lease.go b/internal/utils/lease.go index 22a1649d3..d56f442ab 100644 --- a/internal/utils/lease.go +++ b/internal/utils/lease.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/utils/lease_test.go b/internal/utils/lease_test.go index bdaf0cfdc..88e29d328 100644 --- a/internal/utils/lease_test.go +++ b/internal/utils/lease_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index 8410522f8..02df1097e 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -1,16 +1,6 @@ -// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index f8aacbfaa..e1799788e 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -1,16 +1,6 @@ -// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index e317982ad..6541b7615 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/store.go b/internal/utils/store.go index 4334041d3..fc1eb7695 100644 --- a/internal/utils/store.go +++ b/internal/utils/store.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils diff --git a/internal/utils/store_test.go b/internal/utils/store_test.go index c502f9893..61ee0e912 100644 --- a/internal/utils/store_test.go +++ b/internal/utils/store_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/utils_suite_test.go b/internal/utils/utils_suite_test.go index 9f242798f..7faf50c41 100644 --- a/internal/utils/utils_suite_test.go +++ b/internal/utils/utils_suite_test.go @@ -1,16 +1,6 @@ -// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package utils_test diff --git a/internal/version/version.go b/internal/version/version.go index d13d533ec..641a6b146 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -1,16 +1,6 @@ -// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package version diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/sentinel/config.go index c6fbac0cd..f25d3b44f 100644 --- a/internal/webhook/sentinel/config.go +++ b/internal/webhook/sentinel/config.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package sentinel import ( diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 06e86dea6..5a9d7995b 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package sentinel import ( diff --git a/internal/webhook/sentinel/register.go b/internal/webhook/sentinel/register.go index bae075786..01bb85e12 100644 --- a/internal/webhook/sentinel/register.go +++ b/internal/webhook/sentinel/register.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package sentinel import ( diff --git a/test/it/controller/assets/assets.go b/test/it/controller/assets/assets.go index dd8de0485..3c805dc1f 100644 --- a/test/it/controller/assets/assets.go +++ b/test/it/controller/assets/assets.go @@ -1,16 +1,6 @@ -// Copyright 2023 SAP SE or an SAP affiliate company +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package assets diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index b8cbe1e1c..81d224c06 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index e61813823..6c12a3a34 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index bd44333bf..f873d4950 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package setup import ( diff --git a/test/utils/client.go b/test/utils/client.go index f7568d679..fc675990b 100644 --- a/test/utils/client.go +++ b/test/utils/client.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/test/utils/constants.go b/test/utils/constants.go index 3963b5372..6a300ec64 100644 --- a/test/utils/constants.go +++ b/test/utils/constants.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils const ( diff --git a/test/utils/errors.go b/test/utils/errors.go index 3a66e7879..e08b420ec 100644 --- a/test/utils/errors.go +++ b/test/utils/errors.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/test/utils/imagevector.go b/test/utils/imagevector.go index e3a1670cf..5a9dd73ed 100644 --- a/test/utils/imagevector.go +++ b/test/utils/imagevector.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/test/utils/matcher.go b/test/utils/matcher.go index abffd0b48..1e8d5af0e 100644 --- a/test/utils/matcher.go +++ b/test/utils/matcher.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( From f5b2f7c0babdc57b0d32c3432c99a27797af6a0c Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 14 Mar 2024 17:15:57 +0530 Subject: [PATCH 101/235] removed controller and pkg folders, adapted packages --- Makefile | 22 +- api/validation/etcd.go | 2 +- controllers/compaction/config.go | 85 - controllers/compaction/metrics.go | 102 - controllers/compaction/reconciler.go | 486 ----- controllers/compaction/register.go | 35 - controllers/config.go | 171 -- controllers/custodian/config.go | 69 - controllers/custodian/reconciler.go | 121 -- controllers/custodian/register.go | 46 - controllers/etcd/config.go | 59 - controllers/etcd/etcd_suite_test.go | 17 - controllers/etcd/reconciler.go | 416 ----- controllers/etcd/register.go | 52 - controllers/etcdcopybackupstask/config.go | 53 - .../etcdcopybackupstask_suite_test.go | 21 - controllers/etcdcopybackupstask/reconciler.go | 541 ------ .../etcdcopybackupstask/reconciler_test.go | 938 ---------- controllers/etcdcopybackupstask/register.go | 32 - controllers/manager.go | 128 -- controllers/predicate/predicate.go | 211 --- controllers/predicate/predicate_suite_test.go | 17 - controllers/predicate/predicate_test.go | 658 ------- controllers/secret/config.go | 34 - controllers/secret/reconciler.go | 106 -- controllers/secret/reconciler_test.go | 230 --- controllers/secret/register.go | 40 - controllers/secret/secret_suite_test.go | 21 - controllers/utils/reconciler.go | 19 - controllers/utils/utils_suite_test.go | 17 - controllers/utils/validator.go | 40 - controllers/utils/validator_test.go | 43 - internal/controller/compaction/config.go | 2 +- internal/controller/compaction/register.go | 2 +- .../controller/etcdcopybackupstask/config.go | 4 +- .../etcdcopybackupstask/reconciler.go | 2 +- internal/controller/predicate/predicate.go | 174 -- .../controller/predicate/predicate_test.go | 366 +--- internal/controller/secret/config.go | 2 +- internal/controller/secret/reconciler.go | 2 +- internal/controller/secret/register.go | 2 +- internal/health/condition/builder.go | 2 +- internal/health/condition/builder_test.go | 2 +- .../condition/check_backup_ready_test.go | 2 +- internal/mapper/etcd_to_secret_test.go | 2 +- internal/mapper/statefulset_to_etcd.go | 66 - internal/mapper/statefulset_to_etcd_test.go | 115 -- main.go | 2 +- pkg/client/kubernetes/types.go | 25 - pkg/common/constants.go | 56 - pkg/component/etcd/configmap/configmap.go | 184 -- .../etcd/configmap/configmap_suite_test.go | 17 - .../etcd/configmap/configmap_test.go | 233 --- pkg/component/etcd/configmap/values.go | 54 - pkg/component/etcd/configmap/values_helper.go | 64 - pkg/component/etcd/lease/lease.go | 85 - pkg/component/etcd/lease/lease_member.go | 82 - pkg/component/etcd/lease/lease_snapshot.go | 30 - pkg/component/etcd/lease/lease_suite_test.go | 17 - pkg/component/etcd/lease/lease_test.go | 286 --- pkg/component/etcd/lease/values.go | 25 - pkg/component/etcd/lease/values_helper.go | 22 - .../poddisruptionbudget.go | 74 - .../poddisruptionbudget_suite_test.go | 17 - .../poddisruptionbudget_test.go | 226 --- .../etcd/poddisruptionbudget/values.go | 31 - .../etcd/poddisruptionbudget/values_helper.go | 41 - .../poddisruptionbudget/values_helper_test.go | 49 - pkg/component/etcd/role/role.go | 55 - pkg/component/etcd/role/role_suite_test.go | 17 - pkg/component/etcd/role/role_test.go | 189 -- pkg/component/etcd/role/values.go | 24 - pkg/component/etcd/role/values_helper.go | 37 - pkg/component/etcd/role/values_helper_test.go | 69 - pkg/component/etcd/rolebinding/rolebinding.go | 66 - .../rolebinding/rolebinding_suite_test.go | 17 - .../etcd/rolebinding/rolebinding_test.go | 165 -- pkg/component/etcd/rolebinding/values.go | 24 - .../etcd/rolebinding/values_helper.go | 21 - .../etcd/rolebinding/values_helper_test.go | 49 - pkg/component/etcd/service/service.go | 65 - pkg/component/etcd/service/service_client.go | 50 - pkg/component/etcd/service/service_peer.go | 37 - .../etcd/service/service_suite_test.go | 17 - pkg/component/etcd/service/service_test.go | 200 -- pkg/component/etcd/service/values.go | 33 - pkg/component/etcd/service/values_helper.go | 42 - .../etcd/service/values_helper_test.go | 208 --- .../etcd/serviceaccount/serviceaccount.go | 56 - .../serviceaccount_suite_test.go | 17 - .../serviceaccount/serviceaccount_test.go | 162 -- pkg/component/etcd/serviceaccount/values.go | 21 - .../etcd/serviceaccount/values_helper.go | 20 - .../etcd/serviceaccount/values_helper_test.go | 63 - pkg/component/etcd/statefulset/statefulset.go | 948 ---------- .../statefulset/statefulset_suite_test.go | 17 - .../etcd/statefulset/statefulset_test.go | 1093 ----------- pkg/component/etcd/statefulset/values.go | 126 -- .../etcd/statefulset/values_helper.go | 349 ---- pkg/features/features.go | 35 - pkg/health/condition/builder.go | 131 -- pkg/health/condition/builder_test.go | 177 -- pkg/health/condition/check_all_members.go | 60 - .../condition/check_all_members_test.go | 119 -- pkg/health/condition/check_backup_ready.go | 145 -- .../condition/check_backup_ready_test.go | 268 --- pkg/health/condition/check_ready.go | 62 - pkg/health/condition/check_ready_test.go | 130 -- pkg/health/condition/condition_suite_test.go | 17 - pkg/health/condition/types.go | 48 - pkg/health/etcdmember/builder.go | 105 -- pkg/health/etcdmember/builder_test.go | 177 -- pkg/health/etcdmember/check_ready.go | 147 -- pkg/health/etcdmember/check_ready_test.go | 393 ---- .../etcdmember/etcdmember_suite_test.go | 17 - pkg/health/etcdmember/types.go | 53 - pkg/health/status/check.go | 139 -- pkg/health/status/check_test.go | 271 --- pkg/health/status/status_suite_test.go | 17 - pkg/mapper/etcd_to_secret.go | 72 - pkg/mapper/etcd_to_secret_test.go | 105 -- pkg/mapper/mapper_suite_test.go | 17 - pkg/mapper/statefulset_to_etcd.go | 66 - pkg/mapper/statefulset_to_etcd_test.go | 115 -- pkg/metrics/metrics.go | 110 -- pkg/metrics/metrics_suite_test.go | 17 - pkg/metrics/metrics_test.go | 80 - pkg/mock/controller-runtime/client/doc.go | 6 - pkg/mock/controller-runtime/client/mocks.go | 518 ------ pkg/utils/envvar.go | 116 -- pkg/utils/image.go | 73 - pkg/utils/image_test.go | 162 -- pkg/utils/lease.go | 64 - pkg/utils/miscellaneous.go | 88 - pkg/utils/miscellaneous_test.go | 44 - pkg/utils/statefulset.go | 54 - pkg/utils/statefulset_test.go | 111 -- pkg/utils/store.go | 100 - pkg/utils/store_test.go | 108 -- pkg/utils/utils_suite_test.go | 17 - pkg/version/version.go | 14 - test/e2e/etcd_multi_node_test.go | 2 +- test/e2e/utils.go | 2 +- test/integration/controllers/assets/assets.go | 2 +- .../compaction/compaction_suite_test.go | 2 +- .../controllers/compaction/reconciler_test.go | 4 +- .../custodian/custodian_suite_test.go | 58 - .../controllers/custodian/reconciler_test.go | 191 -- .../controllers/etcd/etcd_suite_test.go | 61 - .../controllers/etcd/reconciler_test.go | 1649 ----------------- .../etcdcopybackupstask_suite_test.go | 2 +- .../etcdcopybackupstask/reconciler_test.go | 4 +- .../controllers/secret/reconciler_test.go | 2 +- .../controllers/secret/secret_suite_test.go | 2 +- 154 files changed, 40 insertions(+), 17963 deletions(-) delete mode 100644 controllers/compaction/config.go delete mode 100644 controllers/compaction/metrics.go delete mode 100644 controllers/compaction/reconciler.go delete mode 100644 controllers/compaction/register.go delete mode 100644 controllers/config.go delete mode 100644 controllers/custodian/config.go delete mode 100644 controllers/custodian/reconciler.go delete mode 100644 controllers/custodian/register.go delete mode 100644 controllers/etcd/config.go delete mode 100644 controllers/etcd/etcd_suite_test.go delete mode 100644 controllers/etcd/reconciler.go delete mode 100644 controllers/etcd/register.go delete mode 100644 controllers/etcdcopybackupstask/config.go delete mode 100644 controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go delete mode 100644 controllers/etcdcopybackupstask/reconciler.go delete mode 100644 controllers/etcdcopybackupstask/reconciler_test.go delete mode 100644 controllers/etcdcopybackupstask/register.go delete mode 100644 controllers/manager.go delete mode 100644 controllers/predicate/predicate.go delete mode 100644 controllers/predicate/predicate_suite_test.go delete mode 100644 controllers/predicate/predicate_test.go delete mode 100644 controllers/secret/config.go delete mode 100644 controllers/secret/reconciler.go delete mode 100644 controllers/secret/reconciler_test.go delete mode 100644 controllers/secret/register.go delete mode 100644 controllers/secret/secret_suite_test.go delete mode 100644 controllers/utils/reconciler.go delete mode 100644 controllers/utils/utils_suite_test.go delete mode 100644 controllers/utils/validator.go delete mode 100644 controllers/utils/validator_test.go delete mode 100644 internal/mapper/statefulset_to_etcd.go delete mode 100644 internal/mapper/statefulset_to_etcd_test.go delete mode 100644 pkg/client/kubernetes/types.go delete mode 100644 pkg/common/constants.go delete mode 100644 pkg/component/etcd/configmap/configmap.go delete mode 100644 pkg/component/etcd/configmap/configmap_suite_test.go delete mode 100644 pkg/component/etcd/configmap/configmap_test.go delete mode 100644 pkg/component/etcd/configmap/values.go delete mode 100644 pkg/component/etcd/configmap/values_helper.go delete mode 100644 pkg/component/etcd/lease/lease.go delete mode 100644 pkg/component/etcd/lease/lease_member.go delete mode 100644 pkg/component/etcd/lease/lease_snapshot.go delete mode 100644 pkg/component/etcd/lease/lease_suite_test.go delete mode 100644 pkg/component/etcd/lease/lease_test.go delete mode 100644 pkg/component/etcd/lease/values.go delete mode 100644 pkg/component/etcd/lease/values_helper.go delete mode 100644 pkg/component/etcd/poddisruptionbudget/poddisruptionbudget.go delete mode 100644 pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_suite_test.go delete mode 100644 pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_test.go delete mode 100644 pkg/component/etcd/poddisruptionbudget/values.go delete mode 100644 pkg/component/etcd/poddisruptionbudget/values_helper.go delete mode 100644 pkg/component/etcd/poddisruptionbudget/values_helper_test.go delete mode 100644 pkg/component/etcd/role/role.go delete mode 100644 pkg/component/etcd/role/role_suite_test.go delete mode 100644 pkg/component/etcd/role/role_test.go delete mode 100644 pkg/component/etcd/role/values.go delete mode 100644 pkg/component/etcd/role/values_helper.go delete mode 100644 pkg/component/etcd/role/values_helper_test.go delete mode 100644 pkg/component/etcd/rolebinding/rolebinding.go delete mode 100644 pkg/component/etcd/rolebinding/rolebinding_suite_test.go delete mode 100644 pkg/component/etcd/rolebinding/rolebinding_test.go delete mode 100644 pkg/component/etcd/rolebinding/values.go delete mode 100644 pkg/component/etcd/rolebinding/values_helper.go delete mode 100644 pkg/component/etcd/rolebinding/values_helper_test.go delete mode 100644 pkg/component/etcd/service/service.go delete mode 100644 pkg/component/etcd/service/service_client.go delete mode 100644 pkg/component/etcd/service/service_peer.go delete mode 100644 pkg/component/etcd/service/service_suite_test.go delete mode 100644 pkg/component/etcd/service/service_test.go delete mode 100644 pkg/component/etcd/service/values.go delete mode 100644 pkg/component/etcd/service/values_helper.go delete mode 100644 pkg/component/etcd/service/values_helper_test.go delete mode 100644 pkg/component/etcd/serviceaccount/serviceaccount.go delete mode 100644 pkg/component/etcd/serviceaccount/serviceaccount_suite_test.go delete mode 100644 pkg/component/etcd/serviceaccount/serviceaccount_test.go delete mode 100644 pkg/component/etcd/serviceaccount/values.go delete mode 100644 pkg/component/etcd/serviceaccount/values_helper.go delete mode 100644 pkg/component/etcd/serviceaccount/values_helper_test.go delete mode 100644 pkg/component/etcd/statefulset/statefulset.go delete mode 100644 pkg/component/etcd/statefulset/statefulset_suite_test.go delete mode 100644 pkg/component/etcd/statefulset/statefulset_test.go delete mode 100644 pkg/component/etcd/statefulset/values.go delete mode 100644 pkg/component/etcd/statefulset/values_helper.go delete mode 100644 pkg/features/features.go delete mode 100644 pkg/health/condition/builder.go delete mode 100644 pkg/health/condition/builder_test.go delete mode 100644 pkg/health/condition/check_all_members.go delete mode 100644 pkg/health/condition/check_all_members_test.go delete mode 100644 pkg/health/condition/check_backup_ready.go delete mode 100644 pkg/health/condition/check_backup_ready_test.go delete mode 100644 pkg/health/condition/check_ready.go delete mode 100644 pkg/health/condition/check_ready_test.go delete mode 100644 pkg/health/condition/condition_suite_test.go delete mode 100644 pkg/health/condition/types.go delete mode 100644 pkg/health/etcdmember/builder.go delete mode 100644 pkg/health/etcdmember/builder_test.go delete mode 100644 pkg/health/etcdmember/check_ready.go delete mode 100644 pkg/health/etcdmember/check_ready_test.go delete mode 100644 pkg/health/etcdmember/etcdmember_suite_test.go delete mode 100644 pkg/health/etcdmember/types.go delete mode 100644 pkg/health/status/check.go delete mode 100644 pkg/health/status/check_test.go delete mode 100644 pkg/health/status/status_suite_test.go delete mode 100644 pkg/mapper/etcd_to_secret.go delete mode 100644 pkg/mapper/etcd_to_secret_test.go delete mode 100644 pkg/mapper/mapper_suite_test.go delete mode 100644 pkg/mapper/statefulset_to_etcd.go delete mode 100644 pkg/mapper/statefulset_to_etcd_test.go delete mode 100644 pkg/metrics/metrics.go delete mode 100644 pkg/metrics/metrics_suite_test.go delete mode 100644 pkg/metrics/metrics_test.go delete mode 100644 pkg/mock/controller-runtime/client/doc.go delete mode 100644 pkg/mock/controller-runtime/client/mocks.go delete mode 100644 pkg/utils/envvar.go delete mode 100755 pkg/utils/image.go delete mode 100644 pkg/utils/image_test.go delete mode 100644 pkg/utils/lease.go delete mode 100644 pkg/utils/miscellaneous.go delete mode 100644 pkg/utils/miscellaneous_test.go delete mode 100644 pkg/utils/statefulset.go delete mode 100644 pkg/utils/statefulset_test.go delete mode 100644 pkg/utils/store.go delete mode 100644 pkg/utils/store_test.go delete mode 100644 pkg/utils/utils_suite_test.go delete mode 100644 pkg/version/version.go delete mode 100644 test/integration/controllers/custodian/custodian_suite_test.go delete mode 100644 test/integration/controllers/custodian/reconciler_test.go delete mode 100644 test/integration/controllers/etcd/etcd_suite_test.go delete mode 100644 test/integration/controllers/etcd/reconciler_test.go diff --git a/Makefile b/Makefile index c14b935d7..6523b9fcb 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,8 @@ IMG ?= ${IMAGE_REPOSITORY}:${IMAGE_BUILD_TAG} ######################################### TOOLS_DIR := $(HACK_DIR)/tools -include $(HACK_DIR)/tools.mk include $(GARDENER_HACK_DIR)/tools.mk +include $(HACK_DIR)/tools.mk .PHONY: tidy tidy: @@ -67,21 +67,20 @@ deploy: $(SKAFFOLD) $(HELM) manifests: $(VGOPATH) $(CONTROLLER_GEN) @GARDENER_HACK_DIR=$(GARDENER_HACK_DIR) VGOPATH=$(VGOPATH) go generate ./config/crd/bases @find "$(REPO_ROOT)/config/crd/bases" -name "*.yaml" -exec cp '{}' "$(REPO_ROOT)/charts/druid/charts/crds/templates/" \; - @controller-gen rbac:roleName=manager-role paths="./controllers/..." + @controller-gen rbac:roleName=manager-role paths="./internal/controller/..." # Run go fmt against code .PHONY: fmt fmt: @env GO111MODULE=on go fmt ./... -.PHONY: clean clean: - @bash $(GARDENER_HACK_DIR)/clean.sh ./api/... ./controllers/... ./pkg/... + @"$(REPO_ROOT)/vendor/github.com/gardener/gardener/hack/clean.sh" ./api/... ./internal/... # Check packages .PHONY: check -check: $(GOLANGCI_LINT) $(GOIMPORTS) fmt manifests - @REPO_ROOT=$(REPO_ROOT) bash $(GARDENER_HACK_DIR)/check.sh --golangci-lint-config=./.golangci.yaml ./api/... ./pkg/... ./controllers/... +check: $(GOLANGCI_LINT) $(GOIMPORTS) set-permissions fmt manifests + @"$(REPO_ROOT)/vendor/github.com/gardener/gardener/hack/check.sh" --golangci-lint-config=./.golangci.yaml ./api/... ./internal/... .PHONY: check-generate check-generate: @@ -107,8 +106,15 @@ docker-push: # Run tests .PHONY: test -test: $(GINKGO) $(SETUP_ENVTEST) - @bash $(HACK_DIR)/test.sh ./api/... ./controllers/... ./pkg/... +test: $(GINKGO) + @"$(REPO_ROOT)/hack/test.sh" ./api/... \ + ./internal/controller/etcdcopybackupstask/... \ + ./internal/controller/predicate/... \ + ./internal/controller/secret/... \ + ./internal/controller/utils/... \ + ./internal/health/... \ + ./internal/mapper/... \ + ./internal/metrics/... .PHONY: test-cov test-cov: $(GINKGO) $(SETUP_ENVTEST) diff --git a/api/validation/etcd.go b/api/validation/etcd.go index cf0d04c81..02e88ce27 100644 --- a/api/validation/etcd.go +++ b/api/validation/etcd.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/utils" apiequality "k8s.io/apimachinery/pkg/api/equality" apivalidation "k8s.io/apimachinery/pkg/api/validation" diff --git a/controllers/compaction/config.go b/controllers/compaction/config.go deleted file mode 100644 index 03bc13633..000000000 --- a/controllers/compaction/config.go +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package compaction - -import ( - "time" - - "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" - - flag "github.com/spf13/pflag" - "k8s.io/component-base/featuregate" -) - -// featureList holds the feature gate names that are relevant for the Compaction Controller. -var featureList = []featuregate.Feature{ - features.UseEtcdWrapper, -} - -const ( - enableBackupCompactionFlagName = "enable-backup-compaction" - workersFlagName = "compaction-workers" - eventsThresholdFlagName = "etcd-events-threshold" - activeDeadlineDurationFlagName = "active-deadline-duration" - metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" - - defaultEnableBackupCompaction = false - defaultCompactionWorkers = 3 - defaultEventsThreshold = 1000000 - defaultActiveDeadlineDuration = 3 * time.Hour - defaultMetricsScrapeWaitDuration = 0 -) - -// Config contains configuration for the Compaction Controller. -type Config struct { - // EnableBackupCompaction specifies whether backup compaction should be enabled. - EnableBackupCompaction bool - // Workers denotes the number of worker threads for the compaction controller. - Workers int - // EventsThreshold denotes total number of etcd events to be reached upon which a backup compaction job is triggered. - EventsThreshold int64 - // ActiveDeadlineDuration is the duration after which a running compaction job will be killed. - ActiveDeadlineDuration time.Duration - // MetricsScrapeWaitDuration is the duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped - MetricsScrapeWaitDuration time.Duration - // FeatureGates contains the feature gates to be used by Compaction Controller. - FeatureGates map[featuregate.Feature]bool -} - -// InitFromFlags initializes the compaction controller config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.BoolVar(&cfg.EnableBackupCompaction, enableBackupCompactionFlagName, defaultEnableBackupCompaction, - "Enable automatic compaction of etcd backups.") - fs.IntVar(&cfg.Workers, workersFlagName, defaultCompactionWorkers, - "Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. Setting this flag to 0 disables the controller.") - fs.Int64Var(&cfg.EventsThreshold, eventsThresholdFlagName, defaultEventsThreshold, - "Total number of etcd events that can be allowed before a backup compaction job is triggered.") - fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, - "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\").") - fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagname, defaultMetricsScrapeWaitDuration, - "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\").") -} - -// Validate validates the config. -func (cfg *Config) Validate() error { - if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, 0, cfg.Workers); err != nil { - return err - } - if err := utils.MustBeGreaterThan(eventsThresholdFlagName, 0, cfg.EventsThreshold); err != nil { - return err - } - return utils.MustBeGreaterThan(activeDeadlineDurationFlagName, 0, cfg.ActiveDeadlineDuration.Seconds()) -} - -// CaptureFeatureActivations captures all feature gates required by the controller into controller config -func (cfg *Config) CaptureFeatureActivations(fg featuregate.FeatureGate) { - if cfg.FeatureGates == nil { - cfg.FeatureGates = make(map[featuregate.Feature]bool) - } - for _, feature := range featureList { - cfg.FeatureGates[feature] = fg.Enabled(feature) - } -} diff --git a/controllers/compaction/metrics.go b/controllers/compaction/metrics.go deleted file mode 100644 index 28cecd891..000000000 --- a/controllers/compaction/metrics.go +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package compaction - -import ( - "time" - - druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" -) - -const ( - namespaceEtcdDruid = "etcddruid" - subsystemCompaction = "compaction" -) - -var ( - baseDuration = 60 * time.Second - // metricJobsTotal is the metric used to count the total number of compaction jobs initiated by compaction controller. - metricJobsTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespaceEtcdDruid, - Subsystem: subsystemCompaction, - Name: "jobs_total", - Help: "Total number of compaction jobs initiated by compaction controller.", - }, - []string{druidmetrics.LabelSucceeded, druidmetrics.EtcdNamespace}, - ) - - // metricJobsCurrent is the metric used to expose currently running compaction job. This metric is important to get a sense of total number of compaction job running in a seed cluster. - metricJobsCurrent = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespaceEtcdDruid, - Subsystem: subsystemCompaction, - Name: "jobs_current", - Help: "Number of currently running comapction job.", - }, - []string{druidmetrics.EtcdNamespace}, - ) - - // metricJobDurationSeconds is the metric used to expose the time taken to finish a compaction job. - metricJobDurationSeconds = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespaceEtcdDruid, - Subsystem: subsystemCompaction, - Name: "job_duration_seconds", - Help: "Total time taken in seconds to finish a running compaction job.", - Buckets: []float64{baseDuration.Seconds(), - 5 * baseDuration.Seconds(), - 10 * baseDuration.Seconds(), - 15 * baseDuration.Seconds(), - 20 * baseDuration.Seconds(), - 30 * baseDuration.Seconds(), - }, - }, - []string{druidmetrics.LabelSucceeded, druidmetrics.EtcdNamespace}, - ) - - // metricNumDeltaEvents is the metric used to expose the total number of etcd events to be compacted by a compaction job. - metricNumDeltaEvents = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespaceEtcdDruid, - Subsystem: subsystemCompaction, - Name: "num_delta_events", - Help: "Total number of etcd events to be compacted by a compaction job.", - }, - []string{druidmetrics.EtcdNamespace}, - ) -) - -func init() { - // metricJobsTotalValues - metricJobsTotalValues := map[string][]string{ - druidmetrics.LabelSucceeded: druidmetrics.DruidLabels[druidmetrics.LabelSucceeded], - druidmetrics.EtcdNamespace: druidmetrics.DruidLabels[druidmetrics.EtcdNamespace], - } - metricJobsTotalCombinations := druidmetrics.GenerateLabelCombinations(metricJobsTotalValues) - for _, combination := range metricJobsTotalCombinations { - metricJobsTotal.With(prometheus.Labels(combination)) - } - - // metricJobDurationSecondsValues - metricJobDurationSecondsValues := map[string][]string{ - druidmetrics.LabelSucceeded: druidmetrics.DruidLabels[druidmetrics.LabelSucceeded], - druidmetrics.EtcdNamespace: druidmetrics.DruidLabels[druidmetrics.EtcdNamespace], - } - compactionJobDurationSecondsCombinations := druidmetrics.GenerateLabelCombinations(metricJobDurationSecondsValues) - for _, combination := range compactionJobDurationSecondsCombinations { - metricJobDurationSeconds.With(prometheus.Labels(combination)) - } - - // Metrics have to be registered to be exposed: - metrics.Registry.MustRegister(metricJobsTotal) - metrics.Registry.MustRegister(metricJobDurationSeconds) - - metrics.Registry.MustRegister(metricJobsCurrent) - - metrics.Registry.MustRegister(metricNumDeltaEvents) -} diff --git a/controllers/compaction/reconciler.go b/controllers/compaction/reconciler.go deleted file mode 100644 index 5795e9040..000000000 --- a/controllers/compaction/reconciler.go +++ /dev/null @@ -1,486 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package compaction - -import ( - "context" - "fmt" - "strconv" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - ctrlutils "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" - druidmetrics "github.com/gardener/etcd-druid/pkg/metrics" - "github.com/gardener/etcd-druid/pkg/utils" - - "github.com/gardener/gardener/pkg/utils/imagevector" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/component-base/featuregate" - "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -const ( - // DefaultETCDQuota is the default etcd quota. - DefaultETCDQuota = 8 * 1024 * 1024 * 1024 // 8Gi -) - -// Reconciler reconciles compaction jobs for Etcd resources. -type Reconciler struct { - client.Client - config *Config - imageVector imagevector.ImageVector - logger logr.Logger -} - -// NewReconciler creates a new reconciler for Compaction -func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := ctrlutils.CreateImageVector() - if err != nil { - return nil, err - } - return NewReconcilerWithImageVector(mgr, config, imageVector), nil -} - -// NewReconcilerWithImageVector creates a new reconciler for Compaction with an ImageVector. -// This constructor will mostly be used by tests. -func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { - return &Reconciler{ - Client: mgr.GetClient(), - config: config, - imageVector: imageVector, - logger: log.Log.WithName("compaction-lease-controller"), - } -} - -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;create;list;watch;update;patch;delete -// +kubebuilder:rbac:groups="",resources=pods,verbs=list;watch;delete;get - -// Reconcile reconciles the compaction job. -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - r.logger.Info("Compaction job reconciliation started") - etcd := &druidv1alpha1.Etcd{} - if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { - if errors.IsNotFound(err) { - // Object not found, return. Created objects are automatically garbage collected. - // For additional cleanup logic use finalizers. - return ctrl.Result{}, nil - } - // Error reading the object - requeue the request. - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } - - if !etcd.DeletionTimestamp.IsZero() || etcd.Spec.Backup.Store == nil { - // Delete compaction job if exists - return r.delete(ctx, r.logger, etcd) - } - - logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) - - return r.reconcileJob(ctx, logger, etcd) -} - -func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { - // Update metrics for currently running compaction job, if any - job := &batchv1.Job{} - if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { - if errors.IsNotFound(err) { - logger.Info("Currently, no compaction job is running in the namespace", "namespace", etcd.Namespace) - } else { - // Error reading the object - requeue the request. - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", etcd.GetCompactionJobName(), etcd.Namespace, err) - } - } - - if job != nil && job.Name != "" { - if !job.DeletionTimestamp.IsZero() { - logger.Info("Job is already in deletion. A new job will be created only if the previous one has been deleted.", "namespace: ", job.Namespace, "name: ", job.Name) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, nil - } - - // Check if there is one active job or not - if job.Status.Active > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) - // Don't need to requeue if the job is currently running - return ctrl.Result{}, nil - } - - // Delete job if the job succeeded - if job.Status.Succeeded > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) - if job.Status.CompletionTime != nil { - metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) - } - if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { - logger.Error(err, "Couldn't delete the successful job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while deleting successful compaction job: %v", err) - } - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() - } - - // Delete job and requeue if the job failed - if job.Status.Failed > 0 { - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(0) - if job.Status.StartTime != nil { - metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) - } - err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) - if err != nil { - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while deleting failed compaction job: %v", err) - } - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, nil - } - } - - // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job - fullLease := &coordinationv1.Lease{} - - if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetFullSnapshotLeaseName()), fullLease); err != nil { - logger.Error(err, "Couldn't fetch full snap lease", "namespace", etcd.Namespace, "name", etcd.GetFullSnapshotLeaseName()) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } - - deltaLease := &coordinationv1.Lease{} - if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetDeltaSnapshotLeaseName()), deltaLease); err != nil { - logger.Error(err, "Couldn't fetch delta snap lease", "namespace", etcd.Namespace, "name", etcd.GetDeltaSnapshotLeaseName()) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } - - // Revisions have not been set yet by etcd-back-restore container. - // Skip further processing as we cannot calculate a revision delta. - if fullLease.Spec.HolderIdentity == nil || deltaLease.Spec.HolderIdentity == nil { - return ctrl.Result{}, nil - } - - full, err := strconv.ParseInt(*fullLease.Spec.HolderIdentity, 10, 64) - if err != nil { - logger.Error(err, "Can't convert holder identity of full snap lease to integer", - "namespace", fullLease.Namespace, "leaseName", fullLease.Name, "holderIdentity", fullLease.Spec.HolderIdentity) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } - - delta, err := strconv.ParseInt(*deltaLease.Spec.HolderIdentity, 10, 64) - if err != nil { - logger.Error(err, "Can't convert holder identity of delta snap lease to integer", - "namespace", deltaLease.Namespace, "leaseName", deltaLease.Name, "holderIdentity", deltaLease.Spec.HolderIdentity) - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, err - } - - diff := delta - full - metricNumDeltaEvents.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(float64(diff)) - - // Reconcile job only when number of accumulated revisions over the last full snapshot is more than the configured threshold value via 'events-threshold' flag - if diff >= r.config.EventsThreshold { - logger.Info("Creating etcd compaction job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) - job, err = r.createCompactionJob(ctx, logger, etcd) - if err != nil { - metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error during compaction job creation: %v", err) - } - metricJobsCurrent.With(prometheus.Labels{druidmetrics.EtcdNamespace: etcd.Namespace}).Set(1) - } - - if job.Name != "" { - logger.Info("Current compaction job status", - "namespace", job.Namespace, "name", job.Name, "succeeded", job.Status.Succeeded) - } - - return ctrl.Result{Requeue: false}, nil -} - -func (r *Reconciler) delete(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { - job := &batchv1.Job{} - err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job) - if err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction job: %v", err) - } - return ctrl.Result{Requeue: false}, nil - } - - if job.DeletionTimestamp == nil { - logger.Info("Deleting job", "namespace", job.Namespace, "name", job.Name) - if err := client.IgnoreNotFound(r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { - return ctrl.Result{ - RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while deleting compaction job: %v", err) - } - } - - logger.Info("No compaction job is running") - return ctrl.Result{ - Requeue: false, - }, nil -} - -func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*batchv1.Job, error) { - activeDeadlineSeconds := r.config.ActiveDeadlineDuration.Seconds() - - _, etcdBackupImage, _, err := utils.GetEtcdImages(etcd, r.imageVector, r.config.FeatureGates[features.UseEtcdWrapper]) - if err != nil { - return nil, fmt.Errorf("couldn't fetch etcd backup image: %v", err) - } - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetCompactionJobName(), - Namespace: etcd.Namespace, - Labels: getLabels(etcd), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: druidv1alpha1.GroupVersion.String(), - BlockOwnerDeletion: pointer.Bool(true), - Controller: pointer.Bool(true), - Kind: "Etcd", - Name: etcd.Name, - UID: etcd.UID, - }, - }, - }, - - Spec: batchv1.JobSpec{ - ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), - Completions: pointer.Int32(1), - BackoffLimit: pointer.Int32(0), - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: etcd.Spec.Annotations, - Labels: getLabels(etcd), - }, - Spec: v1.PodSpec{ - ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), - ServiceAccountName: etcd.GetServiceAccountName(), - RestartPolicy: v1.RestartPolicyNever, - Containers: []v1.Container{{ - Name: "compact-backup", - Image: *etcdBackupImage, - ImagePullPolicy: v1.PullIfNotPresent, - Args: getCompactionJobArgs(etcd, r.config.MetricsScrapeWaitDuration.String()), - }}, - }, - }, - }, - } - - if vms, err := getCompactionJobVolumeMounts(etcd, r.config.FeatureGates); err != nil { - return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", - etcd.Namespace, - etcd.Name, - err) - } else { - job.Spec.Template.Spec.Containers[0].VolumeMounts = vms - } - - if env, err := utils.GetBackupRestoreContainerEnvVars(etcd.Spec.Backup.Store); err != nil { - return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", - etcd.Namespace, - etcd.Name, - err) - } else { - job.Spec.Template.Spec.Containers[0].Env = env - } - - if vm, err := getCompactionJobVolumes(ctx, r.Client, r.logger, etcd); err != nil { - return nil, fmt.Errorf("error creating compaction job in %v for %v : %v", - etcd.Namespace, - etcd.Name, - err) - } else { - job.Spec.Template.Spec.Volumes = vm - } - - if etcd.Spec.Backup.CompactionResources != nil { - job.Spec.Template.Spec.Containers[0].Resources = *etcd.Spec.Backup.CompactionResources - } - - logger.Info("Creating job", "namespace", job.Namespace, "name", job.Name) - err = r.Create(ctx, job) - if err != nil { - return nil, err - } - - //TODO (abdasgupta): Evaluate necessity of claiming object here after creation - return job, nil -} - -func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { - return map[string]string{ - "name": "etcd-backup-compaction", - "instance": etcd.Name, - "gardener.cloud/role": "controlplane", - "networking.gardener.cloud/to-dns": "allowed", - "networking.gardener.cloud/to-private-networks": "allowed", - "networking.gardener.cloud/to-public-networks": "allowed", - } -} -func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featuregate.Feature]bool) ([]v1.VolumeMount, error) { - vms := []v1.VolumeMount{ - { - Name: "etcd-workspace-dir", - MountPath: "/var/etcd/data", - }, - } - - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err != nil { - return vms, fmt.Errorf("storage provider is not recognized while fetching volume mounts") - } - switch provider { - case utils.Local: - if featureMap[features.UseEtcdWrapper] { - vms = append(vms, v1.VolumeMount{ - Name: "host-storage", - MountPath: "/home/nonroot/" + pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), - }) - } else { - vms = append(vms, v1.VolumeMount{ - Name: "host-storage", - MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), - }) - } - case utils.GCS: - vms = append(vms, v1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/.gcp/", - }) - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - vms = append(vms, v1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/etcd-backup/", - }) - } - - return vms, nil -} - -func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr.Logger, etcd *druidv1alpha1.Etcd) ([]v1.Volume, error) { - vs := []v1.Volume{ - { - Name: "etcd-workspace-dir", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, - } - - storeValues := etcd.Spec.Backup.Store - provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) - if err != nil { - return vs, fmt.Errorf("could not recognize storage provider while fetching volumes") - } - switch provider { - case "Local": - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, etcd.Namespace) - if err != nil { - return vs, fmt.Errorf("could not determine host mount path for local provider") - } - - hpt := v1.HostPathDirectory - vs = append(vs, v1.Volume{ - Name: "host-storage", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), - Type: &hpt, - }, - }, - }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: - if storeValues.SecretRef == nil { - return vs, fmt.Errorf("could not configure secretRef for backup store %v", provider) - } - - vs = append(vs, v1.Volume{ - Name: "etcd-backup", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, - }, - }, - }) - } - - return vs, nil -} - -func getCompactionJobArgs(etcd *druidv1alpha1.Etcd, metricsScrapeWaitDuration string) []string { - command := []string{"compact"} - command = append(command, "--data-dir=/var/etcd/data/compaction.etcd") - command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp") - command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") - command = append(command, "--metrics-scrape-wait-duration="+metricsScrapeWaitDuration) - command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) - command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) - - var quota int64 = DefaultETCDQuota - if etcd.Spec.Etcd.Quota != nil { - quota = etcd.Spec.Etcd.Quota.Value() - } - command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) - - if etcd.Spec.Etcd.EtcdDefragTimeout != nil { - command = append(command, "--etcd-defrag-timeout="+etcd.Spec.Etcd.EtcdDefragTimeout.Duration.String()) - } - - backupValues := etcd.Spec.Backup - if backupValues.EtcdSnapshotTimeout != nil { - command = append(command, "--etcd-snapshot-timeout="+backupValues.EtcdSnapshotTimeout.Duration.String()) - } - storeValues := etcd.Spec.Backup.Store - if storeValues != nil { - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) - if err == nil { - command = append(command, "--storage-provider="+provider) - } - - if storeValues.Prefix != "" { - command = append(command, "--store-prefix="+storeValues.Prefix) - } - - if storeValues.Container != nil { - command = append(command, "--store-container="+*(storeValues.Container)) - } - } - - return command -} diff --git a/controllers/compaction/register.go b/controllers/compaction/register.go deleted file mode 100644 index f8dd65db6..000000000 --- a/controllers/compaction/register.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package compaction - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/controllers/predicate" - - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -const controllerName = "compaction-controller" - -// RegisterWithManager registers the Compaction Controller with the given controller manager. -func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { - return ctrl. - NewControllerManagedBy(mgr). - Named(controllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.config.Workers, - }). - For(&druidv1alpha1.Etcd{}). - WithEventFilter(predicate. - Or(druidpredicates.SnapshotRevisionChanged(), - druidpredicates.JobStatusChanged())). - Owns(&coordinationv1.Lease{}). - Owns(&batchv1.Job{}). - Complete(r) -} diff --git a/controllers/config.go b/controllers/config.go deleted file mode 100644 index 66c5375e3..000000000 --- a/controllers/config.go +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package controllers - -import ( - "fmt" - - "github.com/gardener/etcd-druid/controllers/compaction" - "github.com/gardener/etcd-druid/controllers/custodian" - "github.com/gardener/etcd-druid/controllers/etcd" - "github.com/gardener/etcd-druid/controllers/etcdcopybackupstask" - "github.com/gardener/etcd-druid/controllers/secret" - "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" - - flag "github.com/spf13/pflag" - "k8s.io/client-go/tools/leaderelection/resourcelock" - "k8s.io/component-base/featuregate" -) - -const ( - metricsAddrFlagName = "metrics-addr" - enableLeaderElectionFlagName = "enable-leader-election" - leaderElectionIDFlagName = "leader-election-id" - leaderElectionResourceLockFlagName = "leader-election-resource-lock" - ignoreOperationAnnotationFlagName = "ignore-operation-annotation" - disableLeaseCacheFlagName = "disable-lease-cache" - - defaultMetricsAddr = ":8080" - defaultEnableLeaderElection = false - defaultLeaderElectionID = "druid-leader-election" - defaultLeaderElectionResourceLock = resourcelock.LeasesResourceLock - defaultIgnoreOperationAnnotation = false - defaultDisableLeaseCache = false -) - -// LeaderElectionConfig defines the configuration for the leader election for the controller manager. -type LeaderElectionConfig struct { - // EnableLeaderElection specifies whether to enable leader election for controller manager. - EnableLeaderElection bool - // LeaderElectionID is the name of the resource that leader election will use for holding the leader lock. - LeaderElectionID string - // LeaderElectionResourceLock specifies which resource type to use for leader election. - LeaderElectionResourceLock string -} - -// ManagerConfig defines the configuration for the controller manager. -type ManagerConfig struct { - // MetricsAddr is the address the metric endpoint binds to. - MetricsAddr string - LeaderElectionConfig - // DisableLeaseCache specifies whether to disable cache for lease.coordination.k8s.io resources. - DisableLeaseCache bool - // IgnoreOperationAnnotation specifies whether to ignore or honour the operation annotation on resources to be reconciled. - IgnoreOperationAnnotation bool - // FeatureGates contains the feature gates to be used by etcd-druid. - FeatureGates featuregate.MutableFeatureGate - // EtcdControllerConfig is the configuration required for etcd controller. - EtcdControllerConfig *etcd.Config - // CustodianControllerConfig is the configuration required for custodian controller. - CustodianControllerConfig *custodian.Config - // CompactionControllerConfig is the configuration required for compaction controller. - CompactionControllerConfig *compaction.Config - // EtcdCopyBackupsTaskControllerConfig is the configuration required for etcd-copy-backup-tasks controller. - EtcdCopyBackupsTaskControllerConfig *etcdcopybackupstask.Config - // SecretControllerConfig is the configuration required for secret controller. - SecretControllerConfig *secret.Config -} - -// InitFromFlags initializes the controller manager config from the provided CLI flag set. -func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { - flag.StringVar(&cfg.MetricsAddr, metricsAddrFlagName, defaultMetricsAddr, ""+ - "The address the metric endpoint binds to.") - flag.BoolVar(&cfg.EnableLeaderElection, enableLeaderElectionFlagName, defaultEnableLeaderElection, - "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&cfg.LeaderElectionID, leaderElectionIDFlagName, defaultLeaderElectionID, - "Name of the resource that leader election will use for holding the leader lock") - flag.StringVar(&cfg.LeaderElectionResourceLock, leaderElectionResourceLockFlagName, defaultLeaderElectionResourceLock, - "Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.") - flag.BoolVar(&cfg.DisableLeaseCache, disableLeaseCacheFlagName, defaultDisableLeaseCache, - "Disable cache for lease.coordination.k8s.io resources.") - flag.BoolVar(&cfg.IgnoreOperationAnnotation, ignoreOperationAnnotationFlagName, defaultIgnoreOperationAnnotation, - "Specifies whether to ignore or honour the operation annotation on resources to be reconciled.") - - if err := cfg.initFeatureGates(fs); err != nil { - return err - } - - cfg.EtcdControllerConfig = &etcd.Config{} - etcd.InitFromFlags(fs, cfg.EtcdControllerConfig) - - cfg.CustodianControllerConfig = &custodian.Config{} - custodian.InitFromFlags(fs, cfg.CustodianControllerConfig) - - cfg.CompactionControllerConfig = &compaction.Config{} - compaction.InitFromFlags(fs, cfg.CompactionControllerConfig) - - cfg.EtcdCopyBackupsTaskControllerConfig = &etcdcopybackupstask.Config{} - etcdcopybackupstask.InitFromFlags(fs, cfg.EtcdCopyBackupsTaskControllerConfig) - - cfg.SecretControllerConfig = &secret.Config{} - secret.InitFromFlags(fs, cfg.SecretControllerConfig) - - return nil -} - -// initFeatureGates initializes feature gates from the provided CLI flag set. -func (cfg *ManagerConfig) initFeatureGates(fs *flag.FlagSet) error { - featureGates := featuregate.NewFeatureGate() - if err := featureGates.Add(features.GetDefaultFeatures()); err != nil { - return fmt.Errorf("error adding features to the featuregate: %v", err) - } - featureGates.AddFlag(fs) - - cfg.FeatureGates = featureGates - - return nil -} - -// populateControllersFeatureGates adds relevant feature gates to every controller configuration -func (cfg *ManagerConfig) populateControllersFeatureGates() { - // Feature gates populated only for controllers that use feature gates - - // Add etcd controller feature gates - cfg.EtcdControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) - - // Add compaction controller feature gates - cfg.CompactionControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) - - // Add etcd-copy-backups-task controller feature gates - cfg.EtcdCopyBackupsTaskControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) -} - -// Validate validates the controller manager config. -func (cfg *ManagerConfig) Validate() error { - if err := utils.ShouldBeOneOfAllowedValues("LeaderElectionResourceLock", getAllowedLeaderElectionResourceLocks(), cfg.LeaderElectionResourceLock); err != nil { - return err - } - if err := cfg.EtcdControllerConfig.Validate(); err != nil { - return err - } - - if err := cfg.CustodianControllerConfig.Validate(); err != nil { - return err - } - - if err := cfg.CompactionControllerConfig.Validate(); err != nil { - return err - } - - if err := cfg.EtcdCopyBackupsTaskControllerConfig.Validate(); err != nil { - return err - } - - return cfg.SecretControllerConfig.Validate() -} - -// getAllowedLeaderElectionResourceLocks gives the resource names that can be used for leader election. -// TODO: This should be changed as lease is the default choice now for leader election. There is no need -// to provide other options. Should be handled as part of a different Issue/PR. -func getAllowedLeaderElectionResourceLocks() []string { - return []string{ - "endpoints", - "configmaps", - "leases", - "endpointsleases", - "configmapsleases", - } -} diff --git a/controllers/custodian/config.go b/controllers/custodian/config.go deleted file mode 100644 index fe8c0775c..000000000 --- a/controllers/custodian/config.go +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package custodian - -import ( - "time" - - "github.com/gardener/etcd-druid/controllers/utils" - - flag "github.com/spf13/pflag" -) - -const ( - workersFlagName = "custodian-workers" - syncPeriodFlagName = "custodian-sync-period" - etcdMemberNotReadyThresholdFlagName = "etcd-member-notready-threshold" - etcdMemberUnknownThresholdFlagName = "etcd-member-unknown-threshold" - - defaultCustodianWorkers = 3 - defaultCustodianSyncPeriod = 30 * time.Second - defaultEtcdMemberNotReadyThreshold = 5 * time.Minute - defaultEtcdMemberUnknownThreshold = 1 * time.Minute -) - -// Config contains configuration for the Custodian Controller. -type Config struct { - // Workers denotes the number of worker threads for the custodian controller. - Workers int - // SyncPeriod is the duration after which re-enqueuing happens. - SyncPeriod time.Duration - // EtcdMember holds configuration related to etcd members. - EtcdMember EtcdMemberConfig -} - -// EtcdMemberConfig holds configuration related to etcd members. -type EtcdMemberConfig struct { - // NotReadyThreshold is the duration after which an etcd member's state is considered `NotReady`. - NotReadyThreshold time.Duration - // UnknownThreshold is the duration after which an etcd member's state is considered `Unknown`. - UnknownThreshold time.Duration -} - -// InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.IntVar(&cfg.Workers, workersFlagName, defaultCustodianWorkers, - "Number of worker threads for the custodian controller.") - fs.DurationVar(&cfg.SyncPeriod, syncPeriodFlagName, defaultCustodianSyncPeriod, - "Sync period of the custodian controller.") - fs.DurationVar(&cfg.EtcdMember.NotReadyThreshold, etcdMemberNotReadyThresholdFlagName, defaultEtcdMemberNotReadyThreshold, - "Threshold after which an etcd member is considered not ready if the status was unknown before.") - fs.DurationVar(&cfg.EtcdMember.UnknownThreshold, etcdMemberUnknownThresholdFlagName, defaultEtcdMemberUnknownThreshold, - "Threshold after which an etcd member is considered unknown.") -} - -// Validate validates the config. -func (cfg *Config) Validate() error { - if err := utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers); err != nil { - return err - } - if err := utils.MustBeGreaterThan(syncPeriodFlagName, 0, cfg.SyncPeriod); err != nil { - return err - } - if err := utils.MustBeGreaterThan(etcdMemberNotReadyThresholdFlagName, 0, cfg.EtcdMember.NotReadyThreshold); err != nil { - return err - } - return utils.MustBeGreaterThan(etcdMemberUnknownThresholdFlagName, 0, cfg.EtcdMember.UnknownThreshold) -} diff --git a/controllers/custodian/reconciler.go b/controllers/custodian/reconciler.go deleted file mode 100644 index 22b9a1f5f..000000000 --- a/controllers/custodian/reconciler.go +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package custodian - -import ( - "context" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - - "github.com/gardener/etcd-druid/pkg/health/status" - "github.com/gardener/etcd-druid/pkg/utils" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -// Reconciler reconciles status of Etcd object -type Reconciler struct { - client.Client - scheme *runtime.Scheme - config *Config - logger logr.Logger - restConfig *rest.Config -} - -// NewReconciler creates a new reconciler for Custodian. -func NewReconciler(mgr manager.Manager, config *Config) *Reconciler { - return &Reconciler{ - Client: mgr.GetClient(), - scheme: mgr.GetScheme(), - config: config, - restConfig: mgr.GetConfig(), - logger: log.Log.WithName(controllerName), - } -} - -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds/status,verbs=get;update;patch;create - -// Reconcile reconciles the etcd. -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - r.logger.Info("Custodian controller reconciliation started") - etcd := &druidv1alpha1.Etcd{} - if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { - if errors.IsNotFound(err) { - // Object not found, return. Created objects are automatically garbage collected. - // For additional cleanup logic use finalizers. - return ctrl.Result{}, nil - } - // Error reading the object - requeue the request. - return ctrl.Result{}, err - } - - logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) - - if etcd.Status.LastError != nil && *etcd.Status.LastError != "" { - logger.Info("Requeue item because of last error", "namespace", etcd.Namespace, "name", etcd.Name, "lastError", *etcd.Status.LastError) - return ctrl.Result{ - RequeueAfter: 30 * time.Second, - }, nil - } - - statusCheck := status.NewChecker(r.Client, r.config.EtcdMember.NotReadyThreshold, r.config.EtcdMember.UnknownThreshold) - if err := statusCheck.Check(ctx, logger, etcd); err != nil { - logger.Error(err, "Error executing status checks") - return ctrl.Result{}, err - } - - sts, err := utils.GetStatefulSet(ctx, r.Client, etcd) - if err != nil { - return ctrl.Result{}, err - } - - if err := r.updateEtcdStatus(ctx, logger, etcd, sts); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: r.config.SyncPeriod}, nil -} - -func (r *Reconciler) updateEtcdStatus(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) error { - logger.Info("Updating etcd status with statefulset information", "namespace", etcd.Namespace, "name", etcd.Name) - - if sts != nil { - etcd.Status.Etcd = &druidv1alpha1.CrossVersionObjectReference{ - APIVersion: sts.APIVersion, - Kind: sts.Kind, - Name: sts.Name, - } - - ready, _ := utils.IsStatefulSetReady(etcd.Spec.Replicas, sts) - - // To be changed once we have multiple replicas. - etcd.Status.CurrentReplicas = sts.Status.CurrentReplicas - etcd.Status.ReadyReplicas = sts.Status.ReadyReplicas - etcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas - etcd.Status.Replicas = sts.Status.CurrentReplicas - etcd.Status.Ready = &ready - logger.Info("ETCD status updated for statefulset", "namespace", etcd.Namespace, "name", etcd.Name, - "currentReplicas", sts.Status.CurrentReplicas, "readyReplicas", sts.Status.ReadyReplicas, "updatedReplicas", sts.Status.UpdatedReplicas) - } else { - etcd.Status.CurrentReplicas = 0 - etcd.Status.ReadyReplicas = 0 - etcd.Status.UpdatedReplicas = 0 - - etcd.Status.Ready = pointer.Bool(false) - } - - return r.Client.Status().Update(ctx, etcd) -} diff --git a/controllers/custodian/register.go b/controllers/custodian/register.go deleted file mode 100644 index a4bf2af0e..000000000 --- a/controllers/custodian/register.go +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package custodian - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/controllers/predicate" - druidmapper "github.com/gardener/etcd-druid/pkg/mapper" - - "github.com/gardener/gardener/pkg/controllerutils/mapper" - appsv1 "k8s.io/api/apps/v1" - ctrl "sigs.k8s.io/controller-runtime" - ctrlbuilder "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -const controllerName = "custodian-controller" - -// RegisterWithManager registers the Custodian Controller with the given controller manager. -func (r *Reconciler) RegisterWithManager(ctx context.Context, mgr ctrl.Manager, ignoreOperationAnnotation bool) error { - c, err := ctrl. - NewControllerManagedBy(mgr). - Named(controllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.config.Workers, - }). - For( - &druidv1alpha1.Etcd{}, - ctrlbuilder.WithPredicates(druidpredicates.EtcdReconciliationFinished(ignoreOperationAnnotation)), - ). - Build(r) - if err != nil { - return err - } - - return c.Watch( - source.Kind(mgr.GetCache(), &appsv1.StatefulSet{}), - mapper.EnqueueRequestsFrom(ctx, mgr.GetCache(), druidmapper.StatefulSetToEtcd(ctx, mgr.GetClient()), mapper.UpdateWithNew, c.GetLogger()), - druidpredicates.StatefulSetStatusChange(), - ) -} diff --git a/controllers/etcd/config.go b/controllers/etcd/config.go deleted file mode 100644 index 821a34453..000000000 --- a/controllers/etcd/config.go +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcd - -import ( - "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" - - flag "github.com/spf13/pflag" - "k8s.io/component-base/featuregate" -) - -const ( - workersFlagName = "workers" - disableEtcdServiceAccountAutomountFlagName = "disable-etcd-serviceaccount-automount" - - defaultWorkers = 3 - defaultDisableEtcdServiceAccountAutomount = false -) - -// featureList holds the feature gate names that are relevant for the Etcd Controller. -var featureList = []featuregate.Feature{ - features.UseEtcdWrapper, -} - -// Config defines the configuration for the Etcd Controller. -type Config struct { - // Workers is the number of workers concurrently processing reconciliation requests. - Workers int - // DisableEtcdServiceAccountAutomount controls the auto-mounting of service account token for etcd statefulsets. - DisableEtcdServiceAccountAutomount bool - // FeatureGates contains the feature gates to be used by Etcd Controller. - FeatureGates map[featuregate.Feature]bool -} - -// InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, - "Number of worker threads of the etcd controller.") - fs.BoolVar(&cfg.DisableEtcdServiceAccountAutomount, disableEtcdServiceAccountAutomountFlagName, defaultDisableEtcdServiceAccountAutomount, - "If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd statefulsets.") -} - -// Validate validates the config. -func (cfg *Config) Validate() error { - return utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers) -} - -// CaptureFeatureActivations captures all feature gates required by the controller into controller config -func (cfg *Config) CaptureFeatureActivations(fg featuregate.FeatureGate) { - if cfg.FeatureGates == nil { - cfg.FeatureGates = make(map[featuregate.Feature]bool) - } - for _, feature := range featureList { - cfg.FeatureGates[feature] = fg.Enabled(feature) - } -} diff --git a/controllers/etcd/etcd_suite_test.go b/controllers/etcd/etcd_suite_test.go deleted file mode 100644 index 29cf977e4..000000000 --- a/controllers/etcd/etcd_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcd - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestEtcdController(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Etcd Controller Suite") -} diff --git a/controllers/etcd/reconciler.go b/controllers/etcd/reconciler.go deleted file mode 100644 index 10fb06e7b..000000000 --- a/controllers/etcd/reconciler.go +++ /dev/null @@ -1,416 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcd - -import ( - "context" - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - ctrlutils "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/common" - componentconfigmap "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" - componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" - componentpdb "github.com/gardener/etcd-druid/pkg/component/etcd/poddisruptionbudget" - componentrole "github.com/gardener/etcd-druid/pkg/component/etcd/role" - componentrolebinding "github.com/gardener/etcd-druid/pkg/component/etcd/rolebinding" - componentservice "github.com/gardener/etcd-druid/pkg/component/etcd/service" - componentserviceaccount "github.com/gardener/etcd-druid/pkg/component/etcd/serviceaccount" - componentsts "github.com/gardener/etcd-druid/pkg/component/etcd/statefulset" - "github.com/gardener/etcd-druid/pkg/features" - druidutils "github.com/gardener/etcd-druid/pkg/utils" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - gardenercomponent "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/imagevector" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -const ( - // IgnoreReconciliationAnnotation is an annotation set by an operator in order to stop reconciliation. - IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" -) - -// Reconciler reconciles Etcd resources. -type Reconciler struct { - client.Client - scheme *runtime.Scheme - config *Config - recorder record.EventRecorder - restConfig *rest.Config - imageVector imagevector.ImageVector - logger logr.Logger -} - -// NewReconciler creates a new reconciler for Etcd. -func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := ctrlutils.CreateImageVector() - if err != nil { - return nil, err - } - return NewReconcilerWithImageVector(mgr, config, imageVector) -} - -// NewReconcilerWithImageVector creates a new reconciler for Etcd with an ImageVector. -// This constructor will mostly be used by tests. -func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) (*Reconciler, error) { - return &Reconciler{ - Client: mgr.GetClient(), - scheme: mgr.GetScheme(), - config: config, - recorder: mgr.GetEventRecorderFor(controllerName), - restConfig: mgr.GetConfig(), - imageVector: imageVector, - logger: log.Log.WithName(controllerName), - }, nil -} - -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds/finalizers,verbs=get;update;patch;create -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete;deletecollection -// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=events,verbs=create;get;list;watch;patch;update - -// Reconcile reconciles Etcd resources. -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - r.logger.Info("ETCD controller reconciliation started") - etcd := &druidv1alpha1.Etcd{} - if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { - if apierrors.IsNotFound(err) { - // Object not found, return. Created objects are automatically garbage collected. - // For additional cleanup logic use finalizers. - return ctrl.Result{}, nil - } - // Error reading the object - requeue the request. - return ctrl.Result{}, err - } - - if !etcd.DeletionTimestamp.IsZero() { - return r.delete(ctx, etcd) - } - - if _, ok := etcd.Annotations[IgnoreReconciliationAnnotation]; ok { - r.recorder.Eventf( - etcd, - corev1.EventTypeWarning, - "ReconciliationIgnored", - "reconciliation of %s/%s is ignored by etcd-druid due to the presence of annotation %s on the etcd resource", - etcd.Namespace, - etcd.Name, - IgnoreReconciliationAnnotation, - ) - - return ctrl.Result{ - Requeue: false, - }, nil - } - - return r.reconcile(ctx, etcd) -} - -// reconcileResult captures the result of a reconciliation run. -type reconcileResult struct { - svcName *string - sts *appsv1.StatefulSet - err error -} - -func (r *Reconciler) reconcile(ctx context.Context, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { - logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String(), "operation", "reconcile") - - // Add Finalizers to Etcd - if finalizers := sets.NewString(etcd.Finalizers...); !finalizers.Has(common.FinalizerName) { - logger.Info("Adding finalizer", "namespace", etcd.Namespace, "name", etcd.Name, "finalizerName", common.FinalizerName) - if err := controllerutils.AddFinalizers(ctx, r.Client, etcd, common.FinalizerName); err != nil { - if err := r.updateEtcdErrorStatus(ctx, etcd, reconcileResult{err: err}); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - return ctrl.Result{ - Requeue: true, - }, err - } - } - - etcd, err := r.updateEtcdStatusAsNotReady(ctx, etcd) - if err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{ - Requeue: true, - }, err - } - - if err = r.removeOperationAnnotation(ctx, logger, etcd); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{ - Requeue: true, - }, err - } - - result := r.reconcileEtcd(ctx, logger, etcd) - if result.err != nil { - if updateEtcdErr := r.updateEtcdErrorStatus(ctx, etcd, result); updateEtcdErr != nil { - logger.Error(updateEtcdErr, "Error during reconciling ETCD") - return ctrl.Result{ - Requeue: true, - }, updateEtcdErr - } - return ctrl.Result{ - Requeue: true, - }, result.err - } - if err := r.updateEtcdStatus(ctx, etcd, result); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - return ctrl.Result{ - Requeue: false, - }, nil -} - -func (r *Reconciler) delete(ctx context.Context, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { - logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String(), "operation", "delete") - logger.Info("Starting deletion operation", "namespace", etcd.Namespace, "name", etcd.Name) - - stsDeployer := gardenercomponent.OpDestroyAndWait(componentsts.New(r.Client, logger, componentsts.Values{Name: etcd.Name, Namespace: etcd.Namespace}, r.config.FeatureGates)) - if err := stsDeployer.Destroy(ctx); err != nil { - if err = r.updateEtcdErrorStatus(ctx, etcd, reconcileResult{err: err}); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - return ctrl.Result{ - Requeue: true, - }, err - } - - leaseDeployer := componentlease.New(r.Client, logger, etcd.Namespace, componentlease.GenerateValues(etcd)) - if err := leaseDeployer.Destroy(ctx); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - cmDeployer := componentconfigmap.New(r.Client, etcd.Namespace, componentconfigmap.GenerateValues(etcd)) - if err := cmDeployer.Destroy(ctx); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - pdbValues := componentpdb.GenerateValues(etcd) - pdbDeployer := componentpdb.New(r.Client, etcd.Namespace, &pdbValues) - if err := pdbDeployer.Destroy(ctx); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - saValues := componentserviceaccount.GenerateValues(etcd, r.config.DisableEtcdServiceAccountAutomount) - saDeployer := componentserviceaccount.New(r.Client, saValues) - if err := saDeployer.Destroy(ctx); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - roleValues := componentrole.GenerateValues(etcd) - roleDeployer := componentrole.New(r.Client, roleValues) - if err := roleDeployer.Destroy(ctx); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - roleBindingValues := componentrolebinding.GenerateValues(etcd) - roleBindingDeployer := componentrolebinding.New(r.Client, roleBindingValues) - if err := roleBindingDeployer.Destroy(ctx); err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - - if sets.NewString(etcd.Finalizers...).Has(common.FinalizerName) { - logger.Info("Removing finalizer", "namespace", etcd.Namespace, "name", etcd.Name, "finalizerName", common.FinalizerName) - if err := controllerutils.RemoveFinalizers(ctx, r.Client, etcd, common.FinalizerName); client.IgnoreNotFound(err) != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - } - logger.Info("Deleted etcd successfully", "namespace", etcd.Namespace, "name", etcd.Name) - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) reconcileResult { - // Check if Spec.Replicas is odd or even. - // TODO(timuthy): The following checks should rather be part of a validation. Also re-enqueuing doesn't make sense in case the values are invalid. - if etcd.Spec.Replicas > 1 && etcd.Spec.Replicas&1 == 0 { - return reconcileResult{err: fmt.Errorf("Spec.Replicas should not be even number: %d", etcd.Spec.Replicas)} - } - - etcdImage, etcdBackupImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, r.imageVector, r.config.FeatureGates[features.UseEtcdWrapper]) - if err != nil { - return reconcileResult{err: err} - } - - leaseValues := componentlease.GenerateValues(etcd) - leaseDeployer := componentlease.New(r.Client, logger, etcd.Namespace, leaseValues) - if err := leaseDeployer.Deploy(ctx); err != nil { - return reconcileResult{err: err} - } - - serviceValues := componentservice.GenerateValues(etcd) - serviceDeployer := componentservice.New(r.Client, etcd.Namespace, serviceValues) - if err := serviceDeployer.Deploy(ctx); err != nil { - return reconcileResult{err: err} - } - - configMapValues := componentconfigmap.GenerateValues(etcd) - cmDeployer := componentconfigmap.New(r.Client, etcd.Namespace, configMapValues) - if err := cmDeployer.Deploy(ctx); err != nil { - return reconcileResult{err: err} - } - - pdbValues := componentpdb.GenerateValues(etcd) - pdbDeployer := componentpdb.New(r.Client, etcd.Namespace, &pdbValues) - if err := pdbDeployer.Deploy(ctx); err != nil { - return reconcileResult{err: err} - } - - saValues := componentserviceaccount.GenerateValues(etcd, r.config.DisableEtcdServiceAccountAutomount) - saDeployer := componentserviceaccount.New(r.Client, saValues) - err = saDeployer.Deploy(ctx) - if err != nil { - return reconcileResult{err: err} - } - - roleValues := componentrole.GenerateValues(etcd) - roleDeployer := componentrole.New(r.Client, roleValues) - err = roleDeployer.Deploy(ctx) - if err != nil { - return reconcileResult{err: err} - } - - roleBindingValues := componentrolebinding.GenerateValues(etcd) - roleBindingDeployer := componentrolebinding.New(r.Client, roleBindingValues) - err = roleBindingDeployer.Deploy(ctx) - if err != nil { - return reconcileResult{err: err} - } - - peerTLSEnabled, err := druidutils.IsPeerURLTLSEnabled(ctx, r.Client, etcd.Namespace, etcd.Name, logger) - if err != nil { - return reconcileResult{err: err} - } - - peerUrlTLSChangedToEnabled := isPeerTLSChangedToEnabled(peerTLSEnabled, configMapValues) - statefulSetValues, err := componentsts.GenerateValues( - etcd, - &serviceValues.ClientPort, - &serviceValues.PeerPort, - &serviceValues.BackupPort, - *etcdImage, - *etcdBackupImage, - *initContainerImage, - map[string]string{ - "checksum/etcd-configmap": configMapValues.ConfigMapChecksum, - }, peerUrlTLSChangedToEnabled, - r.config.FeatureGates[features.UseEtcdWrapper], - ) - if err != nil { - return reconcileResult{err: err} - } - - // Create an OpWaiter because after the deployment we want to wait until the StatefulSet is ready. - var ( - stsDeployer = componentsts.New(r.Client, logger, *statefulSetValues, r.config.FeatureGates) - deployWaiter = gardenercomponent.OpWait(stsDeployer) - ) - - if err = deployWaiter.Deploy(ctx); err != nil { - return reconcileResult{err: err} - } - - sts, err := stsDeployer.Get(ctx) - return reconcileResult{svcName: &serviceValues.ClientServiceName, sts: sts, err: err} -} - -func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, configMapValues *componentconfigmap.Values) bool { - if peerTLSEnabledStatusFromMembers { - return false - } - return configMapValues.PeerUrlTLS != nil -} - -func (r *Reconciler) updateEtcdErrorStatus(ctx context.Context, etcd *druidv1alpha1.Etcd, result reconcileResult) error { - lastErrStr := result.err.Error() - etcd.Status.LastError = &lastErrStr - etcd.Status.ObservedGeneration = &etcd.Generation - if result.sts != nil { - ready, _ := druidutils.IsStatefulSetReady(etcd.Spec.Replicas, result.sts) - etcd.Status.Ready = &ready - etcd.Status.Replicas = pointer.Int32Deref(result.sts.Spec.Replicas, 0) - } - - return r.Client.Status().Update(ctx, etcd) -} - -func (r *Reconciler) updateEtcdStatus(ctx context.Context, etcd *druidv1alpha1.Etcd, result reconcileResult) error { - if result.sts != nil { - ready, _ := druidutils.IsStatefulSetReady(etcd.Spec.Replicas, result.sts) - etcd.Status.Ready = &ready - etcd.Status.Replicas = pointer.Int32Deref(result.sts.Spec.Replicas, 0) - } - etcd.Status.ServiceName = result.svcName - etcd.Status.LastError = nil - etcd.Status.ObservedGeneration = &etcd.Generation - - return r.Client.Status().Update(ctx, etcd) -} - -func (r *Reconciler) removeOperationAnnotation(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { - if _, ok := etcd.Annotations[v1beta1constants.GardenerOperation]; ok { - logger.Info("Removing operation annotation", "namespace", etcd.Namespace, "name", etcd.Name, "annotation", v1beta1constants.GardenerOperation) - withOpAnnotation := etcd.DeepCopy() - delete(etcd.Annotations, v1beta1constants.GardenerOperation) - return r.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)) - } - return nil -} - -func (r *Reconciler) updateEtcdStatusAsNotReady(ctx context.Context, etcd *druidv1alpha1.Etcd) (*druidv1alpha1.Etcd, error) { - etcd.Status.Ready = nil - etcd.Status.ReadyReplicas = 0 - - return etcd, r.Client.Status().Update(ctx, etcd) -} diff --git a/controllers/etcd/register.go b/controllers/etcd/register.go deleted file mode 100644 index 55c72749d..000000000 --- a/controllers/etcd/register.go +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcd - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/controllers/predicate" - - predicateutils "github.com/gardener/gardener/pkg/controllerutils/predicate" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -const controllerName = "etcd-controller" - -// RegisterWithManager registers the Etcd Controller with the given controller manager. -func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager, ignoreOperationAnnotation bool) error { - builder := ctrl. - NewControllerManagedBy(mgr). - Named(controllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.config.Workers, - }). - WithEventFilter(BuildPredicate(ignoreOperationAnnotation)). - For(&druidv1alpha1.Etcd{}) - - if ignoreOperationAnnotation { - builder = builder.Owns(&corev1.Service{}). - Owns(&corev1.ConfigMap{}). - Owns(&appsv1.StatefulSet{}) - } - - return builder.Complete(r) -} - -// BuildPredicate builds the predicates used by Etcd controller. -func BuildPredicate(ignoreOperationAnnotation bool) predicate.Predicate { - if ignoreOperationAnnotation { - return predicate.GenerationChangedPredicate{} - } - - return predicate.Or( - druidpredicates.HasOperationAnnotation(), - druidpredicates.LastOperationNotSuccessful(), - predicateutils.IsDeleting(), - ) -} diff --git a/controllers/etcdcopybackupstask/config.go b/controllers/etcdcopybackupstask/config.go deleted file mode 100644 index 9bde160db..000000000 --- a/controllers/etcdcopybackupstask/config.go +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdcopybackupstask - -import ( - "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" - - flag "github.com/spf13/pflag" - "k8s.io/component-base/featuregate" -) - -// featureList holds the feature gate names that are relevant for the EtcdCopyBackupTask Controller. -var featureList = []featuregate.Feature{ - features.UseEtcdWrapper, -} - -const ( - workersFlagName = "etcd-copy-backups-task-workers" - - defaultWorkers = 3 -) - -// Config defines the configuration for the EtcdCopyBackupsTask Controller. -type Config struct { - // Workers is the number of workers concurrently processing reconciliation requests. - Workers int - // FeatureGates contains the feature gates to be used by EtcdCopyBackupTask Controller. - FeatureGates map[featuregate.Feature]bool -} - -// InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, - "Number of worker threads for the etcdcopybackupstask controller.") -} - -// Validate validates the config. -func (cfg *Config) Validate() error { - return utils.MustBeGreaterThanOrEqualTo(workersFlagName, 0, cfg.Workers) -} - -// CaptureFeatureActivations captures all feature gates required by the controller into controller config -func (cfg *Config) CaptureFeatureActivations(fg featuregate.FeatureGate) { - if cfg.FeatureGates == nil { - cfg.FeatureGates = make(map[featuregate.Feature]bool) - } - for _, feature := range featureList { - cfg.FeatureGates[feature] = fg.Enabled(feature) - } -} diff --git a/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go b/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go deleted file mode 100644 index 52515953e..000000000 --- a/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdcopybackupstask - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestEtcdCopyBackupsTaskController(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs( - t, - "EtcdCopyBackupsTask Controller Suite", - ) -} diff --git a/controllers/etcdcopybackupstask/reconciler.go b/controllers/etcdcopybackupstask/reconciler.go deleted file mode 100644 index a20097654..000000000 --- a/controllers/etcdcopybackupstask/reconciler.go +++ /dev/null @@ -1,541 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdcopybackupstask - -import ( - "context" - "fmt" - "strconv" - "strings" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - ctrlutils "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/features" - "github.com/gardener/etcd-druid/pkg/utils" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/imagevector" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -const ( - sourcePrefix = "source-" - targetPrefix = "target-" -) - -// Reconciler reconciles EtcdCopyBackupsTask object. -type Reconciler struct { - client.Client - Config *Config - imageVector imagevector.ImageVector - logger logr.Logger -} - -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcdcopybackupstasks,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcdcopybackupstasks/status;etcdcopybackupstasks/finalizers,verbs=get;update;patch;create - -// NewReconciler creates a new reconciler for EtcdCopyBackupsTask. -func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := ctrlutils.CreateImageVector() - if err != nil { - return nil, err - } - return NewReconcilerWithImageVector(mgr, config, imageVector), nil -} - -// NewReconcilerWithImageVector creates a new reconciler for EtcdCopyBackupsTask with an ImageVector. -// This constructor will mostly be used by tests. -func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, imageVector imagevector.ImageVector) *Reconciler { - return &Reconciler{ - Client: mgr.GetClient(), - Config: config, - imageVector: imageVector, - logger: log.Log.WithName("etcd-copy-backups-task-controller"), - } -} - -// Reconcile reconciles the EtcdCopyBackupsTask. -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - task := &druidv1alpha1.EtcdCopyBackupsTask{} - if err := r.Get(ctx, req.NamespacedName, task); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - if !task.DeletionTimestamp.IsZero() { - return r.delete(ctx, task) - } - return r.reconcile(ctx, task) -} - -func (r *Reconciler) reconcile(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (result ctrl.Result, err error) { - logger := r.logger.WithValues("etcdCopyBackupsTask", kutil.ObjectName(task), "operation", "reconcile") - - // Ensure finalizer - if !controllerutil.ContainsFinalizer(task, common.FinalizerName) { - logger.V(1).Info("Adding finalizer", "finalizerName", common.FinalizerName) - if err := controllerutils.AddFinalizers(ctx, r.Client, task, common.FinalizerName); err != nil { - return ctrl.Result{}, fmt.Errorf("could not add finalizer: %w", err) - } - } - - var status *druidv1alpha1.EtcdCopyBackupsTaskStatus - defer func() { - // Update status, on failure return the update error unless there is another error - if updateErr := r.updateStatus(ctx, task, status); updateErr != nil && err == nil { - err = fmt.Errorf("could not update status for task {name: %s, namespace: %s} : %w", task.Name, task.Namespace, updateErr) - } - }() - - // Reconcile creation or update - logger.V(1).Info("Reconciling creation or update for etcd-copy-backups-task", "name", task.Name, "namespace", task.Namespace) - if status, err = r.doReconcile(ctx, task, logger); err != nil { - return ctrl.Result{}, fmt.Errorf("could not reconcile creation or update: %w", err) - } - logger.V(1).Info("Creation or update reconciled for etcd-copy-backups-task", "name", task.Name, "namespace", task.Namespace) - - return ctrl.Result{}, nil -} - -func (r *Reconciler) doReconcile(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, logger logr.Logger) (status *druidv1alpha1.EtcdCopyBackupsTaskStatus, err error) { - status = task.Status.DeepCopy() - - var job *batchv1.Job - defer func() { - setStatusDetails(status, task.Generation, job, err) - }() - - // Get job from cluster - job, err = r.getJob(ctx, task) - if err != nil { - return status, err - } - if job != nil { - return status, nil - } - - // create a job object from task - job, err = r.createJobObject(ctx, task) - if err != nil { - return status, err - } - - // Create job - logger.Info("Creating job", "namespace", job.Namespace, "name", job.Name) - if err := r.Create(ctx, job); err != nil { - return status, fmt.Errorf("could not create job %s: %w", kutil.ObjectName(job), err) - } - - return status, nil -} - -func (r *Reconciler) delete(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (result ctrl.Result, err error) { - logger := r.logger.WithValues("task", kutil.ObjectName(task), "operation", "delete") - - // Check finalizer - if !controllerutil.ContainsFinalizer(task, common.FinalizerName) { - logger.V(1).Info("Skipping since finalizer not present", "finalizerName", common.FinalizerName) - return ctrl.Result{}, nil - } - - var status *druidv1alpha1.EtcdCopyBackupsTaskStatus - var removeFinalizer bool - defer func() { - // Only update status if the finalizer is not removed to prevent errors if the object is already gone - if !removeFinalizer { - // Update status, on failure return the update error unless there is another error - if updateErr := r.updateStatus(ctx, task, status); updateErr != nil && err == nil { - err = fmt.Errorf("could not update status: %w", updateErr) - } - } - }() - - // Reconcile deletion - logger.V(1).Info("Reconciling deletion") - if status, removeFinalizer, err = r.doDelete(ctx, task, logger); err != nil { - return ctrl.Result{}, fmt.Errorf("could not reconcile deletion: %w", err) - } - logger.V(1).Info("Deletion reconciled") - - // Remove finalizer if requested - if removeFinalizer { - logger.V(1).Info("Removing finalizer", "finalizerName", common.FinalizerName) - if err := controllerutils.RemoveFinalizers(ctx, r.Client, task, common.FinalizerName); err != nil { - return ctrl.Result{}, fmt.Errorf("could not remove finalizer: %w", err) - } - } - - return ctrl.Result{}, nil -} - -func (r *Reconciler) doDelete(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, logger logr.Logger) (status *druidv1alpha1.EtcdCopyBackupsTaskStatus, removeFinalizer bool, err error) { - status = task.Status.DeepCopy() - - var job *batchv1.Job - defer func() { - setStatusDetails(status, task.Generation, job, err) - }() - - // Get job from cluster - job, err = r.getJob(ctx, task) - if err != nil { - return status, false, err - } - if job == nil { - return status, true, nil - } - - // Delete job if needed - if job.DeletionTimestamp == nil { - logger.Info("Deleting job", "namespace", job.Namespace, "name", job.Name) - if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); client.IgnoreNotFound(err) != nil { - return status, false, fmt.Errorf("could not delete job %s: %w", kutil.ObjectName(job), err) - } - } - - return status, false, nil -} - -func (r *Reconciler) getJob(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (*batchv1.Job, error) { - job := &batchv1.Job{} - if err := r.Get(ctx, kutil.Key(task.Namespace, task.GetJobName()), job); err != nil { - if apierrors.IsNotFound(err) { - return nil, nil - } - return nil, err - } - return job, nil -} - -func (r *Reconciler) updateStatus(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask, status *druidv1alpha1.EtcdCopyBackupsTaskStatus) error { - if status == nil { - return nil - } - patch := client.MergeFromWithOptions(task.DeepCopy(), client.MergeFromWithOptimisticLock{}) - task.Status = *status - return r.Client.Status().Patch(ctx, task, patch) -} - -func setStatusDetails(status *druidv1alpha1.EtcdCopyBackupsTaskStatus, generation int64, job *batchv1.Job, err error) { - status.ObservedGeneration = &generation - if job != nil { - status.Conditions = getConditions(job.Status.Conditions) - } else { - status.Conditions = nil - } - if err != nil { - status.LastError = pointer.String(err.Error()) - } else { - status.LastError = nil - } -} - -func getConditions(jobConditions []batchv1.JobCondition) []druidv1alpha1.Condition { - var conditions []druidv1alpha1.Condition - for _, jobCondition := range jobConditions { - if conditionType := getConditionType(jobCondition.Type); conditionType != "" { - conditions = append(conditions, druidv1alpha1.Condition{ - Type: conditionType, - Status: druidv1alpha1.ConditionStatus(jobCondition.Status), - LastTransitionTime: jobCondition.LastTransitionTime, - LastUpdateTime: jobCondition.LastProbeTime, - Reason: jobCondition.Reason, - Message: jobCondition.Message, - }) - } - } - return conditions -} - -func getConditionType(jobConditionType batchv1.JobConditionType) druidv1alpha1.ConditionType { - switch jobConditionType { - case batchv1.JobComplete: - return druidv1alpha1.EtcdCopyBackupsTaskSucceeded - case batchv1.JobFailed: - return druidv1alpha1.EtcdCopyBackupsTaskFailed - } - return "" -} - -func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.EtcdCopyBackupsTask) (*batchv1.Job, error) { - etcdBackupImage, err := utils.GetEtcdBackupRestoreImage(r.imageVector, r.Config.FeatureGates[features.UseEtcdWrapper]) - if err != nil { - return nil, err - } - - initContainerImage, err := utils.GetInitContainerImage(r.imageVector) - if err != nil { - return nil, err - } - - targetStore := task.Spec.TargetStore - targetProvider, err := utils.StorageProviderFromInfraProvider(targetStore.Provider) - if err != nil { - return nil, err - } - - sourceStore := task.Spec.SourceStore - sourceProvider, err := utils.StorageProviderFromInfraProvider(sourceStore.Provider) - if err != nil { - return nil, err - } - - // Formulate the job's arguments. - args := createJobArgs(task, sourceProvider, targetProvider) - - // Formulate the job environment variables. - env := append(createEnvVarsFromStore(&sourceStore, sourceProvider, "SOURCE_", sourcePrefix), createEnvVarsFromStore(&targetStore, targetProvider, "", "")...) - - // Formulate the job's volume mounts. - volumeMounts := append(createVolumeMountsFromStore(&sourceStore, sourceProvider, sourcePrefix, r.Config.FeatureGates[features.UseEtcdWrapper]), createVolumeMountsFromStore(&targetStore, targetProvider, targetPrefix, r.Config.FeatureGates[features.UseEtcdWrapper])...) - - // Formulate the job's volumes from the source store. - sourceVolumes, err := r.createVolumesFromStore(ctx, &sourceStore, task.Namespace, sourceProvider, sourcePrefix) - if err != nil { - return nil, err - } - - // Formulate the job's volumes from the target store. - targetVolumes, err := r.createVolumesFromStore(ctx, &targetStore, task.Namespace, targetProvider, targetPrefix) - if err != nil { - return nil, err - } - - // Combine the source and target volumes. - volumes := append(sourceVolumes, targetVolumes...) - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: task.GetJobName(), - Namespace: task.Namespace, - Annotations: map[string]string{ - common.GardenerOwnedBy: client.ObjectKeyFromObject(task).String(), - common.GardenerOwnerType: strings.ToLower(task.Kind), - }, - }, - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed, - v1beta1constants.LabelNetworkPolicyToPublicNetworks: v1beta1constants.LabelNetworkPolicyAllowed, - }, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - Containers: []corev1.Container{ - { - Name: "copy-backups", - Image: *etcdBackupImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: args, - Env: env, - VolumeMounts: volumeMounts, - }, - }, - ShareProcessNamespace: pointer.Bool(true), - Volumes: volumes, - }, - }, - }, - } - - if r.Config.FeatureGates[features.UseEtcdWrapper] { - if targetProvider == utils.Local { - // init container to change file permissions of the folders used as store to 65532 (nonroot) - // used only with local provider - job.Spec.Template.Spec.InitContainers = []corev1.Container{ - { - Name: "change-backup-bucket-permissions", - Image: *initContainerImage, - Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("%s%s%s%s", "chown -R 65532:65532 /home/nonroot/", *targetStore.Container, " /home/nonroot/", *sourceStore.Container)}, - VolumeMounts: volumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }, - } - } - job.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ - RunAsGroup: pointer.Int64(65532), - RunAsNonRoot: pointer.Bool(true), - RunAsUser: pointer.Int64(65532), - } - } - - if err := controllerutil.SetControllerReference(task, job, r.Scheme()); err != nil { - return nil, fmt.Errorf("could not set owner reference for job %v: %w", kutil.ObjectName(job), err) - } - return job, nil -} - -func createJobArgs(task *druidv1alpha1.EtcdCopyBackupsTask, sourceObjStoreProvider string, targetObjStoreProvider string) []string { - // Create the initial arguments for the copy-backups job. - args := []string{ - "copy", - "--snapstore-temp-directory=/home/nonroot/data/tmp", - } - - // Formulate the job's arguments. - args = append(args, createJobArgumentFromStore(&task.Spec.TargetStore, targetObjStoreProvider, "")...) - args = append(args, createJobArgumentFromStore(&task.Spec.SourceStore, sourceObjStoreProvider, sourcePrefix)...) - if task.Spec.MaxBackupAge != nil { - args = append(args, "--max-backup-age="+strconv.Itoa(int(*task.Spec.MaxBackupAge))) - } - - if task.Spec.MaxBackups != nil { - args = append(args, "--max-backups-to-copy="+strconv.Itoa(int(*task.Spec.MaxBackups))) - } - - if task.Spec.WaitForFinalSnapshot != nil { - args = append(args, "--wait-for-final-snapshot="+strconv.FormatBool(task.Spec.WaitForFinalSnapshot.Enabled)) - if task.Spec.WaitForFinalSnapshot.Timeout != nil { - args = append(args, "--wait-for-final-snapshot-timeout="+task.Spec.WaitForFinalSnapshot.Timeout.Duration.String()) - } - } - return args -} - -// getVolumeNamePrefix returns the appropriate volume name prefix based on the provided prefix. -// If the provided prefix is "source-", it returns the prefix; otherwise, it returns an empty string. -func getVolumeNamePrefix(prefix string) string { - if prefix == sourcePrefix { - return prefix - } - return "" -} - -// createVolumesFromStore generates a slice of VolumeMounts for an EtcdCopyBackups job based on the given StoreSpec and -// provider. The prefix is used to differentiate between source and target volume. -// This function creates the necessary Volume configurations for various storage providers. -func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1alpha1.StoreSpec, namespace, provider, prefix string) (volumes []corev1.Volume, err error) { - switch provider { - case utils.Local: - hostPathDirectory := corev1.HostPathDirectory - hostPathPrefix, err := utils.GetHostMountPathFromSecretRef(ctx, r.Client, r.logger, store, namespace) - if err != nil { - return nil, err - } - volumes = append(volumes, corev1.Volume{ - Name: prefix + "host-storage", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: hostPathPrefix + "/" + *store.Container, - Type: &hostPathDirectory, - }, - }, - }) - case utils.GCS, utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: - if store.SecretRef == nil { - err = fmt.Errorf("no secretRef is configured for backup %sstore", prefix) - return - } - volumes = append(volumes, corev1.Volume{ - Name: getVolumeNamePrefix(prefix) + "etcd-backup", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: store.SecretRef.Name, - }, - }, - }) - - } - return -} - -// createVolumesFromStore generates a slice of volumes for an EtcdCopyBackups job based on the given StoreSpec, namespace, -// provider, and prefix. The prefix is used to differentiate between source and target volumes. -// This function creates the necessary Volume configurations for various storage providers and returns any errors encountered. -func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volumeMountPrefix string, useEtcdWrapper bool) (volumeMounts []corev1.VolumeMount) { - switch provider { - case utils.Local: - if useEtcdWrapper { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: volumeMountPrefix + "host-storage", - MountPath: "/home/nonroot/" + *store.Container, - }) - } else { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: volumeMountPrefix + "host-storage", - MountPath: *store.Container, - }) - } - case utils.GCS: - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", - MountPath: "/var/." + getVolumeNamePrefix(volumeMountPrefix) + "gcp/", - }) - case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", - MountPath: "/var/" + getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup/", - }) - } - return -} - -// createEnvVarsFromStore generates a slice of environment variables for an EtcdCopyBackups job based on the given StoreSpec, -// storeProvider, prefix, and volumePrefix. The prefix is used to differentiate between source and target environment variables. -// This function creates the necessary environment variables for various storage providers and configurations. The generated -// environment variables include storage container information and provider-specific credentials. -func createEnvVarsFromStore(store *druidv1alpha1.StoreSpec, storeProvider, envKeyPrefix, volumePrefix string) (envVars []corev1.EnvVar) { - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvStorageContainer, *store.Container)) - switch storeProvider { - case utils.S3: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) - case utils.ABS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) - case utils.GCS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, "/var/."+volumePrefix+"gcp/serviceaccount.json")) - case utils.Swift: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) - case utils.OCS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) - case utils.OSS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) - } - return envVars -} - -// createJobArgumentFromStore generates a slice of command-line arguments for a EtcdCopyBackups job based on the given StoreSpec, -// provider, and prefix. The prefix is used to differentiate between source and target command-line arguments. -// This function is used to create the necessary command-line arguments for -// various storage providers and configurations. The generated arguments include storage provider, -// store prefix, and store container information. -func createJobArgumentFromStore(store *druidv1alpha1.StoreSpec, provider, prefix string) (arguments []string) { - if store == nil || len(provider) == 0 { - return - } - argPrefix := "--" + prefix - arguments = append(arguments, argPrefix+"storage-provider="+provider) - - if len(store.Prefix) > 0 { - arguments = append(arguments, argPrefix+"store-prefix="+store.Prefix) - } - - if store.Container != nil && len(*store.Container) > 0 { - arguments = append(arguments, argPrefix+"store-container="+*store.Container) - } - return -} diff --git a/controllers/etcdcopybackupstask/reconciler_test.go b/controllers/etcdcopybackupstask/reconciler_test.go deleted file mode 100644 index fb343c9c2..000000000 --- a/controllers/etcdcopybackupstask/reconciler_test.go +++ /dev/null @@ -1,938 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdcopybackupstask - -import ( - "context" - "fmt" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" - testutils "github.com/gardener/etcd-druid/test/utils" - - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/imagevector" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - gomegatypes "github.com/onsi/gomega/types" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/component-base/featuregate" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var _ = Describe("EtcdCopyBackupsTaskController", func() { - - Describe("#getConditions", func() { - var ( - jobConditions []batchv1.JobCondition - ) - - BeforeEach(func() { - jobConditions = []batchv1.JobCondition{ - { - Type: batchv1.JobComplete, - Status: corev1.ConditionTrue, - }, - { - Type: batchv1.JobFailed, - Status: corev1.ConditionFalse, - }, - } - }) - - It("should get the correct conditions from the job", func() { - conditions := getConditions(jobConditions) - Expect(len(conditions)).To(Equal(len(jobConditions))) - for i, condition := range conditions { - if condition.Type == druidv1alpha1.EtcdCopyBackupsTaskSucceeded { - Expect(jobConditions[i].Type).To(Equal(batchv1.JobComplete)) - } else if condition.Type == druidv1alpha1.EtcdCopyBackupsTaskFailed { - Expect(jobConditions[i].Type).To(Equal(batchv1.JobFailed)) - } else { - Fail("got unexpected condition type") - } - Expect(condition.Status).To(Equal(druidv1alpha1.ConditionStatus(jobConditions[i].Status))) - } - }) - }) - - Describe("#delete", func() { - var ( - ctx = context.Background() - task *druidv1alpha1.EtcdCopyBackupsTask - fakeClient client.WithWatch - r *Reconciler - ) - const ( - testTaskName = "test-etcd-copy-backups-task" - testNamespace = "test-ns" - ) - - Context("delete EtcdCopyBackupsTask object when it exists", func() { - BeforeEach(func() { - task = testutils.CreateEtcdCopyBackupsTask(testTaskName, testNamespace, "aws", false) - - By("Create fake client with task object") - fakeClient = fakeclient.NewClientBuilder(). - WithScheme(kubernetes.Scheme). - WithObjects(task). - WithStatusSubresource(task). - Build() - - By("Ensure that copy backups task is created") - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) - }).Should(Succeed()) - - r = &Reconciler{ - Client: fakeClient, - logger: logr.Discard(), - } - }) - AfterEach(func() { - ensureEtcdCopyBackupsTaskRemoval(ctx, testTaskName, testNamespace, fakeClient) - }) - - It("should not delete task if there is no deletion timestamp set and no finalizer set", func() { - _, err := r.delete(ctx, task) - Expect(err).To(BeNil()) - foundTask := &druidv1alpha1.EtcdCopyBackupsTask{} - Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(task), foundTask)).To(Succeed()) - Expect(client.ObjectKeyFromObject(foundTask)).To(Equal(client.ObjectKeyFromObject(task))) - }) - - It("should remove finalizer and delete task if it does not have a corresponding job", func() { - Expect(controllerutils.AddFinalizers(ctx, fakeClient, task, common.FinalizerName)).To(Succeed()) - // use fakeClient.Delete() to simply add deletionTimestamp to `task` object, - // due to https://github.com/kubernetes-sigs/controller-runtime/pull/2316 - Expect(fakeClient.Delete(ctx, task)).To(Succeed()) - // get the updated object after deletionTimestamp has been added by fakeClient.Delete() call - Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task)).To(Succeed()) - - _, err := r.delete(ctx, task) - Expect(err).To(BeNil()) - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) - }).Should(BeNotFoundError()) - }) - - It("should delete job but not the task for which deletion timestamp and finalizer are set and job is present", func() { - job := testutils.CreateEtcdCopyBackupsJob(testTaskName, testNamespace) - Expect(fakeClient.Create(ctx, job)).To(Succeed()) - Expect(controllerutils.AddFinalizers(ctx, fakeClient, task, common.FinalizerName)).To(Succeed()) - Expect(fakeClient.Delete(ctx, task)).To(Succeed()) - Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task)).To(Succeed()) - - _, err := r.delete(ctx, task) - Expect(err).To(BeNil()) - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(job), job) - }).Should(BeNotFoundError()) - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) - }).Should(BeNil()) - }) - }) - }) - - Describe("#createJobObject", func() { - var ( - reconciler *Reconciler - ctx = context.Background() - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - namespace = "test-ns" - unknownProvider = druidv1alpha1.StorageProvider("unknown") - ) - - BeforeEach(func() { - reconciler = &Reconciler{ - Client: fakeClient, - logger: logr.Discard(), - imageVector: imagevector.ImageVector{ - &imagevector.ImageSource{ - Name: common.BackupRestore, - Repository: "test-repo", - Tag: pointer.String("etcd-test-tag"), - }, - &imagevector.ImageSource{ - Name: common.Alpine, - Repository: "test-repo", - Tag: pointer.String("init-container-test-tag"), - }, - }, - Config: &Config{ - FeatureGates: make(map[featuregate.Feature]bool), - }, - } - }) - - DescribeTable("should create the expected job object with correct metadata, pod template, and containers for a valid input task", - func(taskName string, provider druidv1alpha1.StorageProvider, withOptionalFields bool) { - task := testutils.CreateEtcdCopyBackupsTask(taskName, namespace, provider, withOptionalFields) - errors := testutils.CreateSecrets(ctx, fakeClient, task.Namespace, task.Spec.SourceStore.SecretRef.Name, task.Spec.TargetStore.SecretRef.Name) - Expect(errors).Should(BeNil()) - - job, err := reconciler.createJobObject(ctx, task) - Expect(err).NotTo(HaveOccurred()) - Expect(job).Should(PointTo(matchJob(task, reconciler.imageVector))) - }, - Entry("with #Local provider, without optional fields", - "foo01", druidv1alpha1.StorageProvider("Local"), false), - Entry("with #Local provider, with optional fields", - "foo02", druidv1alpha1.StorageProvider("Local"), true), - Entry("with #S3 storage provider, without optional fields", - "foo03", druidv1alpha1.StorageProvider("aws"), false), - Entry("with #S3 storage provider, with optional fields", - "foo04", druidv1alpha1.StorageProvider("aws"), true), - Entry("with #AZURE storage provider, without optional fields", - "foo05", druidv1alpha1.StorageProvider("azure"), false), - Entry("with #AZURE storage provider, with optional fields", - "foo06", druidv1alpha1.StorageProvider("azure"), true), - Entry("with #GCP storage provider, without optional fields", - "foo07", druidv1alpha1.StorageProvider("gcp"), false), - Entry("with #GCP storage provider, with optional fields", - "foo08", druidv1alpha1.StorageProvider("gcp"), true), - Entry("with #OPENSTACK storage provider, without optional fields", - "foo09", druidv1alpha1.StorageProvider("openstack"), false), - Entry("with #OPENSTACK storage provider, with optional fields", - "foo10", druidv1alpha1.StorageProvider("openstack"), true), - Entry("with #ALICLOUD storage provider, without optional fields", - "foo11", druidv1alpha1.StorageProvider("alicloud"), false), - Entry("with #ALICLOUD storage provider, with optional fields", - "foo12", druidv1alpha1.StorageProvider("alicloud"), true), - ) - - Context("when etcd-backup image is not found", func() { - It("should return error", func() { - reconciler.imageVector = nil - task := testutils.CreateEtcdCopyBackupsTask("test", namespace, "Local", true) - job, err := reconciler.createJobObject(ctx, task) - - Expect(job).To(BeNil()) - Expect(err).To(HaveOccurred()) - }) - }) - - Context("when target store provider is unknown", func() { - It("should return error", func() { - task := testutils.CreateEtcdCopyBackupsTask("test", namespace, "Local", true) - task.Spec.TargetStore.Provider = &unknownProvider - job, err := reconciler.createJobObject(ctx, task) - - Expect(job).To(BeNil()) - Expect(err).To(HaveOccurred()) - }) - }) - - Context("when source store provider is unknown", func() { - It("should return error", func() { - task := testutils.CreateEtcdCopyBackupsTask("test", namespace, "Local", true) - task.Spec.SourceStore.Provider = &unknownProvider - job, err := reconciler.createJobObject(ctx, task) - - Expect(job).To(BeNil()) - Expect(err).To(HaveOccurred()) - }) - }) - }) - - Describe("#createJobArgumentFromStore", func() { - Context("when given a nil store", func() { - It("returns an empty argument slice", func() { - Expect(createJobArgumentFromStore(nil, "provider", "prefix")).To(BeEmpty()) - }) - }) - - Context("when given a empty provider", func() { - It("returns an empty argument slice", func() { - Expect(createJobArgumentFromStore(nil, "", "prefix")).To(BeEmpty()) - }) - }) - - Context("when given a non-nil store", func() { - var ( - store druidv1alpha1.StoreSpec - provider string - prefix string - ) - - BeforeEach(func() { - store = druidv1alpha1.StoreSpec{ - Prefix: "store_prefix", - Container: pointer.String("store_container"), - } - provider = "storage_provider" - prefix = "prefix" - }) - - It("returns a argument slice with provider, prefix, and container information", func() { - expected := []string{ - "--prefixstorage-provider=storage_provider", - "--prefixstore-prefix=store_prefix", - "--prefixstore-container=store_container", - } - Expect(createJobArgumentFromStore(&store, provider, prefix)).To(Equal(expected)) - }) - - It("should return a argument slice with provider and prefix information only when StoreSpec.Container is nil", func() { - expected := []string{ - "--prefixstorage-provider=storage_provider", - "--prefixstore-prefix=store_prefix", - } - store.Container = nil - Expect(createJobArgumentFromStore(&store, provider, prefix)).To(Equal(expected)) - }) - - It("should return a argument slice with provider and container information only when StoreSpec.Prefix is empty", func() { - expected := []string{ - "--prefixstorage-provider=storage_provider", - "--prefixstore-container=store_container", - } - store.Prefix = "" - Expect(createJobArgumentFromStore(&store, provider, prefix)).To(Equal(expected)) - }) - - It("returns an empty argument slice when StoreSpec.Provider is empty", func() { - provider = "" - Expect(createJobArgumentFromStore(&store, provider, prefix)).To(BeEmpty()) - }) - }) - }) - - Describe("#createJobArguments", func() { - var ( - providerLocal = druidv1alpha1.StorageProvider(utils.Local) - providerS3 = druidv1alpha1.StorageProvider(utils.S3) - task *druidv1alpha1.EtcdCopyBackupsTask - expected = []string{ - "copy", - "--snapstore-temp-directory=/home/nonroot/data/tmp", - "--storage-provider=S3", - "--store-prefix=/target", - "--store-container=target-container", - "--source-storage-provider=Local", - "--source-store-prefix=/source", - "--source-store-container=source-container", - } - ) - - BeforeEach(func() { - task = &druidv1alpha1.EtcdCopyBackupsTask{ - Spec: druidv1alpha1.EtcdCopyBackupsTaskSpec{ - SourceStore: druidv1alpha1.StoreSpec{ - Prefix: "/source", - Container: pointer.String("source-container"), - Provider: &providerLocal, - }, - TargetStore: druidv1alpha1.StoreSpec{ - Prefix: "/target", - Container: pointer.String("target-container"), - Provider: &providerS3, - SecretRef: &corev1.SecretReference{ - Name: "test-secret", - }, - }, - }, - } - }) - - It("should create the correct arguments", func() { - arguments := createJobArgs(task, utils.Local, utils.S3) - Expect(arguments).To(Equal(expected)) - }) - - It("should include the max backup age in the arguments", func() { - task.Spec.MaxBackupAge = pointer.Uint32(10) - arguments := createJobArgs(task, utils.Local, utils.S3) - Expect(arguments).To(Equal(append(expected, "--max-backup-age=10"))) - }) - - It("should include the max number of backups in the arguments", func() { - task.Spec.MaxBackups = pointer.Uint32(5) - arguments := createJobArgs(task, utils.Local, utils.S3) - Expect(arguments).To(Equal(append(expected, "--max-backups-to-copy=5"))) - }) - - It("should include the wait for final snapshot in the arguments", func() { - task.Spec.WaitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ - Enabled: true, - } - arguments := createJobArgs(task, utils.Local, utils.S3) - Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true"))) - }) - - It("should include the wait for final snapshot and timeout in the arguments", func() { - task.Spec.WaitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ - Enabled: true, - Timeout: &metav1.Duration{Duration: time.Minute}, - } - arguments := createJobArgs(task, utils.Local, utils.S3) - Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true", "--wait-for-final-snapshot-timeout=1m0s"))) - }) - }) - - Describe("#createEnvVarsFromStore", func() { - var ( - envKeyPrefix = "SOURCE_" - volumePrefix = "source-" - container = "source-container" - storeSpec *druidv1alpha1.StoreSpec - ) - // Loop through different storage providers to test with - for _, p := range []string{ - utils.ABS, - utils.GCS, - utils.S3, - utils.Swift, - utils.OSS, - utils.OCS, - } { - Context(fmt.Sprintf("with provider #%s", p), func() { - provider := p - BeforeEach(func() { - storageProvider := druidv1alpha1.StorageProvider(provider) - storeSpec = &druidv1alpha1.StoreSpec{ - Container: &container, - Provider: &storageProvider, - } - }) - - It("should create the correct env vars", func() { - envVars := createEnvVarsFromStore(storeSpec, provider, envKeyPrefix, volumePrefix) - checkEnvVars(envVars, provider, container, envKeyPrefix, volumePrefix) - - }) - }) - } - Context("with provider #Local", func() { - BeforeEach(func() { - storageProvider := druidv1alpha1.StorageProvider(utils.Local) - storeSpec = &druidv1alpha1.StoreSpec{ - Container: &container, - Provider: &storageProvider, - } - }) - - It("should create the correct env vars", func() { - envVars := createEnvVarsFromStore(storeSpec, utils.Local, envKeyPrefix, volumePrefix) - checkEnvVars(envVars, utils.Local, container, envKeyPrefix, volumePrefix) - - }) - }) - }) - - Describe("#createVolumeMountsFromStore", func() { - var ( - volumeMountPrefix = "source-" - storeSpec *druidv1alpha1.StoreSpec - ) - // Loop through different storage providers to test with - for _, p := range []string{ - utils.Local, - utils.ABS, - utils.GCS, - utils.S3, - utils.Swift, - utils.OSS, - utils.OCS, - } { - Context(fmt.Sprintf("with provider #%s", p), func() { - provider := p - BeforeEach(func() { - storageProvider := druidv1alpha1.StorageProvider(provider) - storeSpec = &druidv1alpha1.StoreSpec{ - Container: pointer.String("source-container"), - Provider: &storageProvider, - } - }) - - It("should create the correct volume mounts", func() { - volumeMounts := createVolumeMountsFromStore(storeSpec, provider, volumeMountPrefix, false) - Expect(volumeMounts).To(HaveLen(1)) - - expectedMountPath := "" - expectedMountName := "" - - switch provider { - case utils.Local: - expectedMountName = volumeMountPrefix + "host-storage" - expectedMountPath = *storeSpec.Container - case utils.GCS: - expectedMountName = volumeMountPrefix + "etcd-backup" - expectedMountPath = "/var/." + volumeMountPrefix + "gcp/" - case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: - expectedMountName = volumeMountPrefix + "etcd-backup" - expectedMountPath = "/var/" + volumeMountPrefix + "etcd-backup/" - default: - Fail(fmt.Sprintf("Unknown provider: %s", provider)) - } - - Expect(volumeMounts[0].Name).To(Equal(expectedMountName)) - Expect(volumeMounts[0].MountPath).To(Equal(expectedMountPath)) - }) - }) - } - }) - - Describe("#createVolumesFromStore", func() { - Context("with provider #Local", func() { - var ( - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - ctx = context.Background() - secret *corev1.Secret - providerLocal = druidv1alpha1.StorageProvider("Local") - namespace = "test-ns" - reconciler = &Reconciler{ - Client: fakeClient, - logger: logr.Discard(), - } - - store = &druidv1alpha1.StoreSpec{ - Container: pointer.String("source-container"), - Prefix: "/tmp", - Provider: &providerLocal, - } - ) - - BeforeEach(func() { - secret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", - Namespace: namespace, - }, - } - }) - - AfterEach(func() { - Expect(fakeClient.Delete(ctx, secret)).To(Succeed()) - }) - - It("should create the correct volumes when secret data hostPath is set", func() { - secret.Data = map[string][]byte{ - utils.EtcdBackupSecretHostPath: []byte("/test/hostPath"), - } - Expect(fakeClient.Create(ctx, secret)).To(Succeed()) - - store.SecretRef = &corev1.SecretReference{Name: secret.Name} - volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(providerLocal), "source-") - Expect(err).NotTo(HaveOccurred()) - Expect(volumes).To(HaveLen(1)) - Expect(volumes[0].Name).To(Equal("source-host-storage")) - - hostPathVolumeSource := volumes[0].VolumeSource.HostPath - Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal("/test/hostPath/" + *store.Container)) - Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) - }) - - It("should create the correct volumes when secret data hostPath is not set", func() { - Expect(fakeClient.Create(ctx, secret)).To(Succeed()) - - store.SecretRef = &corev1.SecretReference{Name: secret.Name} - volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(providerLocal), "source-") - Expect(err).NotTo(HaveOccurred()) - Expect(volumes).To(HaveLen(1)) - Expect(volumes[0].Name).To(Equal("source-host-storage")) - - hostPathVolumeSource := volumes[0].VolumeSource.HostPath - Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal(utils.LocalProviderDefaultMountPath + "/" + *store.Container)) - Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) - }) - - It("should create the correct volumes when store.SecretRef is not referred", func() { - Expect(fakeClient.Create(ctx, secret)).To(Succeed()) - - store.SecretRef = &corev1.SecretReference{Name: secret.Name} - volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(providerLocal), "source-") - Expect(err).NotTo(HaveOccurred()) - Expect(volumes).To(HaveLen(1)) - Expect(volumes[0].Name).To(Equal("source-host-storage")) - - hostPathVolumeSource := volumes[0].VolumeSource.HostPath - Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal(utils.LocalProviderDefaultMountPath + "/" + *store.Container)) - Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) - }) - }) - - Context("with provider", func() { - var ( - storageProvider druidv1alpha1.StorageProvider - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - ctx = context.Background() - secret *corev1.Secret - store *druidv1alpha1.StoreSpec - namespace = "test-ns" - reconciler = &Reconciler{ - Client: fakeClient, - logger: logr.Discard(), - } - ) - - // Loop through different storage providers to test with - for _, p := range []string{ - utils.ABS, - utils.GCS, - utils.S3, - utils.Swift, - utils.OSS, - utils.OCS, - } { - Context(fmt.Sprintf("#%s", p), func() { - BeforeEach(func() { - provider := p - // Set up test variables and create necessary secrets - storageProvider = druidv1alpha1.StorageProvider(provider) - store = &druidv1alpha1.StoreSpec{ - Container: pointer.String("source-container"), - Provider: &storageProvider, - } - secret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret-" + provider, - Namespace: namespace, - }, - } - Expect(fakeClient.Create(ctx, secret)).To(Succeed()) - }) - - AfterEach(func() { - // Clean up secret after each test case - Expect(fakeClient.Delete(ctx, secret)).To(Succeed()) - }) - - It("should create the correct volumes", func() { - // Call the function being tested with a valid secret reference - store.SecretRef = &corev1.SecretReference{Name: secret.Name} - volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(storageProvider), "source-") - Expect(err).NotTo(HaveOccurred()) - Expect(volumes).To(HaveLen(1)) - Expect(volumes[0].Name).To(Equal("source-etcd-backup")) - - // Assert that the volume is created correctly with the expected secret - volumeSource := volumes[0].VolumeSource - Expect(volumeSource).NotTo(BeNil()) - Expect(volumeSource.Secret).NotTo(BeNil()) - Expect(*volumeSource.Secret).To(Equal(corev1.SecretVolumeSource{ - SecretName: store.SecretRef.Name, - })) - }) - - It("should return an error when secret reference is invalid", func() { - // Call the function being tested with an invalid secret reference - volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(storageProvider), "source-") - - // Assert that an error is returned and no volumes are created - Expect(err.Error()).To(Equal("no secretRef is configured for backup source-store")) - Expect(volumes).To(HaveLen(0)) - }) - }) - } - }) - }) -}) - -func ensureEtcdCopyBackupsTaskRemoval(ctx context.Context, name, namespace string, fakeClient client.WithWatch) { - task := &druidv1alpha1.EtcdCopyBackupsTask{} - if err := fakeClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, task); err != nil { - Expect(err).To(BeNotFoundError()) - return - } - - By("Remove any existing finalizers on EtcdCopyBackupsTask") - Expect(controllerutils.RemoveAllFinalizers(ctx, fakeClient, task)).To(Succeed()) - - By("Delete EtcdCopyBackupsTask") - err := fakeClient.Delete(ctx, task) - if err != nil { - Expect(err).Should(BeNotFoundError()) - } - - By("Ensure EtcdCopyBackupsTask is deleted") - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) - }).Should(BeNotFoundError()) -} - -func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefix, volumePrefix string) { - expected := []corev1.EnvVar{ - { - Name: envKeyPrefix + common.EnvStorageContainer, - Value: container, - }} - mapToEnvVarKey := map[string]string{ - utils.S3: envKeyPrefix + common.EnvAWSApplicationCredentials, - utils.ABS: envKeyPrefix + common.EnvAzureApplicationCredentials, - utils.GCS: envKeyPrefix + common.EnvGoogleApplicationCredentials, - utils.Swift: envKeyPrefix + common.EnvOpenstackApplicationCredentials, - utils.OCS: envKeyPrefix + common.EnvOpenshiftApplicationCredentials, - utils.OSS: envKeyPrefix + common.EnvAlicloudApplicationCredentials, - } - switch storeProvider { - case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: - expected = append(expected, corev1.EnvVar{ - Name: mapToEnvVarKey[storeProvider], - Value: "/var/" + volumePrefix + "etcd-backup", - }) - case utils.GCS: - expected = append(expected, corev1.EnvVar{ - Name: mapToEnvVarKey[storeProvider], - Value: "/var/." + volumePrefix + "gcp/serviceaccount.json", - }) - } - Expect(envVars).To(Equal(expected)) -} - -func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.ImageVector) gomegatypes.GomegaMatcher { - sourceProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.SourceStore.Provider) - Expect(err).NotTo(HaveOccurred()) - targetProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) - Expect(err).NotTo(HaveOccurred()) - - images, err := imagevector.FindImages(imageVector, []string{common.BackupRestore}) - Expect(err).NotTo(HaveOccurred()) - backupRestoreImage := images[common.BackupRestore] - - matcher := MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(task.Name + "-worker"), - "Namespace": Equal(task.Namespace), - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", task.Namespace, task.Name)), - "gardener.cloud/owner-type": Equal("etcdcopybackupstask"), - }), - "OwnerReferences": MatchAllElements(testutils.OwnerRefIterator, Elements{ - task.Name: MatchAllFields(Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("EtcdCopyBackupsTask"), - "Name": Equal(task.Name), - "UID": Equal(task.UID), - "Controller": PointTo(Equal(true)), - "BlockOwnerDeletion": PointTo(Equal(true)), - }), - }), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Labels": MatchKeys(IgnoreExtras, Keys{ - "networking.gardener.cloud/to-dns": Equal("allowed"), - "networking.gardener.cloud/to-public-networks": Equal("allowed"), - }), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "RestartPolicy": Equal(corev1.RestartPolicyOnFailure), - "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ - "copy-backups": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("copy-backups"), - "Image": Equal(fmt.Sprintf("%s:%s", backupRestoreImage.Repository, *backupRestoreImage.Tag)), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "Args": MatchAllElements(testutils.CmdIterator, getArgElements(task, sourceProvider, targetProvider)), - "Env": MatchElements(testutils.EnvIterator, IgnoreExtras, getEnvElements(task)), - }), - }), - }), - }), - }), - }) - - return And(matcher, matchJobWithProviders(task, sourceProvider, targetProvider)) -} - -func getArgElements(task *druidv1alpha1.EtcdCopyBackupsTask, sourceProvider, targetProvider string) Elements { - elements := Elements{ - "copy": Equal("copy"), - "--snapstore-temp-directory=/home/nonroot/data/tmp": Equal("--snapstore-temp-directory=/home/nonroot/data/tmp"), - } - if targetProvider != "" { - addEqual(elements, fmt.Sprintf("%s=%s", "--storage-provider", targetProvider)) - } - if task.Spec.TargetStore.Prefix != "" { - addEqual(elements, fmt.Sprintf("%s=%s", "--store-prefix", task.Spec.TargetStore.Prefix)) - } - if task.Spec.TargetStore.Container != nil && *task.Spec.TargetStore.Container != "" { - addEqual(elements, fmt.Sprintf("%s=%s", "--store-container", *task.Spec.TargetStore.Container)) - } - if sourceProvider != "" { - addEqual(elements, fmt.Sprintf("%s=%s", "--source-storage-provider", sourceProvider)) - } - if task.Spec.SourceStore.Prefix != "" { - addEqual(elements, fmt.Sprintf("%s=%s", "--source-store-prefix", task.Spec.SourceStore.Prefix)) - } - if task.Spec.SourceStore.Container != nil && *task.Spec.SourceStore.Container != "" { - addEqual(elements, fmt.Sprintf("%s=%s", "--source-store-container", *task.Spec.SourceStore.Container)) - } - if task.Spec.MaxBackupAge != nil && *task.Spec.MaxBackupAge != 0 { - addEqual(elements, fmt.Sprintf("%s=%d", "--max-backup-age", *task.Spec.MaxBackupAge)) - } - if task.Spec.MaxBackups != nil && *task.Spec.MaxBackups != 0 { - addEqual(elements, fmt.Sprintf("%s=%d", "--max-backups-to-copy", *task.Spec.MaxBackups)) - } - if task.Spec.WaitForFinalSnapshot != nil && task.Spec.WaitForFinalSnapshot.Enabled { - addEqual(elements, fmt.Sprintf("%s=%t", "--wait-for-final-snapshot", task.Spec.WaitForFinalSnapshot.Enabled)) - if task.Spec.WaitForFinalSnapshot.Timeout != nil && task.Spec.WaitForFinalSnapshot.Timeout.Duration != 0 { - addEqual(elements, fmt.Sprintf("%s=%s", "--wait-for-final-snapshot-timeout", task.Spec.WaitForFinalSnapshot.Timeout.Duration.String())) - } - } - return elements -} - -func getEnvElements(task *druidv1alpha1.EtcdCopyBackupsTask) Elements { - elements := Elements{} - if task.Spec.TargetStore.Container != nil && *task.Spec.TargetStore.Container != "" { - elements[common.EnvStorageContainer] = MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*task.Spec.TargetStore.Container), - }) - } - if task.Spec.SourceStore.Container != nil && *task.Spec.SourceStore.Container != "" { - elements[common.EnvSourceStorageContainer] = MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvSourceStorageContainer), - "Value": Equal(*task.Spec.SourceStore.Container), - }) - } - return elements -} - -func matchJobWithProviders(task *druidv1alpha1.EtcdCopyBackupsTask, sourceProvider, targetProvider string) gomegatypes.GomegaMatcher { - matcher := MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ - "copy-backups": MatchFields(IgnoreExtras, Fields{ - "Env": And( - MatchElements(testutils.EnvIterator, IgnoreExtras, getProviderEnvElements(targetProvider, "", "")), - MatchElements(testutils.EnvIterator, IgnoreExtras, getProviderEnvElements(sourceProvider, "SOURCE_", "source-")), - ), - }), - }), - }), - }), - }), - }) - if sourceProvider == "GCS" || targetProvider == "GCS" { - volumeMatcher := MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ - "copy-backups": MatchFields(IgnoreExtras, Fields{ - "VolumeMounts": And( - MatchElements(testutils.VolumeMountIterator, IgnoreExtras, getVolumeMountsElements(targetProvider, "")), - MatchElements(testutils.VolumeMountIterator, IgnoreExtras, getVolumeMountsElements(sourceProvider, "source-")), - ), - }), - }), - "Volumes": And( - MatchElements(testutils.VolumeIterator, IgnoreExtras, getVolumesElements("", &task.Spec.TargetStore)), - MatchElements(testutils.VolumeIterator, IgnoreExtras, getVolumesElements("source-", &task.Spec.SourceStore)), - ), - }), - }), - }), - }) - return And(matcher, volumeMatcher) - } - return matcher -} - -func getProviderEnvElements(storeProvider, prefix, volumePrefix string) Elements { - switch storeProvider { - case "S3": - return Elements{ - prefix + common.EnvAWSApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + common.EnvAWSApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), - }), - } - case "ABS": - return Elements{ - prefix + common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + common.EnvAzureApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), - }), - } - case "GCS": - return Elements{ - prefix + common.EnvGoogleApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + common.EnvGoogleApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/.%sgcp/serviceaccount.json", volumePrefix)), - }), - } - case "Swift": - return Elements{ - prefix + common.EnvOpenstackApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + common.EnvOpenstackApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), - }), - } - case "OSS": - return Elements{ - prefix + common.EnvAlicloudApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + common.EnvAlicloudApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), - }), - } - case "OCS": - return Elements{ - prefix + common.EnvOpenshiftApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(prefix + common.EnvOpenshiftApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), - }), - } - default: - return nil - } -} - -func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { - switch storeProvider { - case "GCS": - return Elements{ - volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), - "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), - }), - } - default: - return Elements{ - volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), - "MountPath": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), - }), - } - } -} - -func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Elements { - return Elements{ - volumePrefix + "etcd-backup": MatchAllFields(Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(store.SecretRef.Name), - })), - }), - }), - } -} - -func addEqual(elements Elements, s string) { - elements[s] = Equal(s) -} diff --git a/controllers/etcdcopybackupstask/register.go b/controllers/etcdcopybackupstask/register.go deleted file mode 100644 index 7eb8172be..000000000 --- a/controllers/etcdcopybackupstask/register.go +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdcopybackupstask - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - - batchv1 "k8s.io/api/batch/v1" - ctrl "sigs.k8s.io/controller-runtime" - ctrlbuilder "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -const controllerName = "etcdcopybackupstask-controller" - -// RegisterWithManager registers the EtcdCopyBackupsTask Controller with the given controller manager. -func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - Named(controllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.Config.Workers, - }). - For( - &druidv1alpha1.EtcdCopyBackupsTask{}, - ctrlbuilder.WithPredicates(predicate.GenerationChangedPredicate{}), - ). - Owns(&batchv1.Job{}). - Complete(r) -} diff --git a/controllers/manager.go b/controllers/manager.go deleted file mode 100644 index b404d7a63..000000000 --- a/controllers/manager.go +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package controllers - -import ( - "context" - "time" - - "github.com/gardener/etcd-druid/controllers/compaction" - "github.com/gardener/etcd-druid/controllers/custodian" - "github.com/gardener/etcd-druid/controllers/etcd" - "github.com/gardener/etcd-druid/controllers/etcdcopybackupstask" - "github.com/gardener/etcd-druid/controllers/secret" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - - coordinationv1 "k8s.io/api/coordination/v1" - coordinationv1beta1 "k8s.io/api/coordination/v1beta1" - corev1 "k8s.io/api/core/v1" - eventsv1 "k8s.io/api/events/v1" - eventsv1beta1 "k8s.io/api/events/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" -) - -var ( - defaultTimeout = time.Minute -) - -// CreateManagerWithControllers creates a controller manager and adds all the controllers to the controller-manager using the passed in ManagerConfig. -func CreateManagerWithControllers(config *ManagerConfig) (ctrl.Manager, error) { - var ( - err error - mgr ctrl.Manager - ) - - config.populateControllersFeatureGates() - - if mgr, err = createManager(config); err != nil { - return nil, err - } - if err = registerControllersWithManager(mgr, config); err != nil { - return nil, err - } - - return mgr, nil -} - -func createManager(config *ManagerConfig) (ctrl.Manager, error) { - // TODO: this can be removed once we have an improved informer, see https://github.com/gardener/etcd-druid/issues/215 - // list of objects which should not be cached. - uncachedObjects := []client.Object{ - &corev1.Event{}, - &eventsv1beta1.Event{}, - &eventsv1.Event{}, - } - - if config.DisableLeaseCache { - uncachedObjects = append(uncachedObjects, &coordinationv1.Lease{}, &coordinationv1beta1.Lease{}) - } - - return ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Client: client.Options{ - Cache: &client.CacheOptions{ - DisableFor: uncachedObjects, - }, - }, - Scheme: kubernetes.Scheme, - Metrics: metricsserver.Options{ - BindAddress: config.MetricsAddr, - }, - LeaderElection: config.EnableLeaderElection, - LeaderElectionID: config.LeaderElectionID, - LeaderElectionResourceLock: config.LeaderElectionResourceLock, - }) -} - -func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) error { - var err error - - // Add etcd reconciler to the manager - etcdReconciler, err := etcd.NewReconciler(mgr, config.EtcdControllerConfig) - if err != nil { - return err - } - if err = etcdReconciler.RegisterWithManager(mgr, config.IgnoreOperationAnnotation); err != nil { - return err - } - - // Add custodian reconciler to the manager - custodianReconciler := custodian.NewReconciler(mgr, config.CustodianControllerConfig) - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) - defer cancel() - if err = custodianReconciler.RegisterWithManager(ctx, mgr, config.IgnoreOperationAnnotation); err != nil { - return err - } - - // Add compaction reconciler to the manager if the CLI flag enable-backup-compaction is true. - if config.CompactionControllerConfig.EnableBackupCompaction { - compactionReconciler, err := compaction.NewReconciler(mgr, config.CompactionControllerConfig) - if err != nil { - return err - } - if err = compactionReconciler.RegisterWithManager(mgr); err != nil { - return err - } - } - - // Add etcd-copy-backups-task reconciler to the manager - etcdCopyBackupsTaskReconciler, err := etcdcopybackupstask.NewReconciler(mgr, config.EtcdCopyBackupsTaskControllerConfig) - if err != nil { - return err - } - if err = etcdCopyBackupsTaskReconciler.RegisterWithManager(mgr); err != nil { - return err - } - - // Add secret reconciler to the manager - return secret.NewReconciler( - mgr, - config.SecretControllerConfig, - ).RegisterWithManager(ctx, mgr) -} diff --git a/controllers/predicate/predicate.go b/controllers/predicate/predicate.go deleted file mode 100644 index befd1d05a..000000000 --- a/controllers/predicate/predicate.go +++ /dev/null @@ -1,211 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package predicate - -import ( - "reflect" - "strings" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -) - -func hasOperationAnnotation(obj client.Object) bool { - return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile -} - -// HasOperationAnnotation is a predicate for the operation annotation. -func HasOperationAnnotation() predicate.Predicate { - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return hasOperationAnnotation(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return hasOperationAnnotation(event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return hasOperationAnnotation(event.Object) - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return true - }, - } -} - -// LastOperationNotSuccessful is a predicate for unsuccessful last operations for creation events. -func LastOperationNotSuccessful() predicate.Predicate { - operationNotSucceeded := func(obj runtime.Object) bool { - etcd, ok := obj.(*druidv1alpha1.Etcd) - if !ok { - return false - } - if etcd.Status.LastError != nil { - return true - } - return false - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return operationNotSucceeded(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return operationNotSucceeded(event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return operationNotSucceeded(event.Object) - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return operationNotSucceeded(event.Object) - }, - } -} - -// StatefulSetStatusChange is a predicate for status changes of `StatefulSet` resources. -func StatefulSetStatusChange() predicate.Predicate { - statusChange := func(objOld, objNew client.Object) bool { - stsOld, ok := objOld.(*appsv1.StatefulSet) - if !ok { - return false - } - stsNew, ok := objNew.(*appsv1.StatefulSet) - if !ok { - return false - } - return !apiequality.Semantic.DeepEqual(stsOld.Status, stsNew.Status) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return true - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return statusChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return true - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return true - }, - } -} - -// EtcdReconciliationFinished is a predicate to use for etcd resources whose reconciliation has finished. -func EtcdReconciliationFinished(ignoreOperationAnnotation bool) predicate.Predicate { - reconciliationFinished := func(obj client.Object) bool { - etcd, ok := obj.(*druidv1alpha1.Etcd) - if !ok { - return false - } - - if etcd.Status.ObservedGeneration == nil { - return false - } - - condition := *etcd.Status.ObservedGeneration == etcd.Generation - - if !ignoreOperationAnnotation { - condition = condition && !hasOperationAnnotation(etcd) - } - - return condition - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return reconciliationFinished(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return reconciliationFinished(event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return reconciliationFinished(event.Object) - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} - -// SnapshotRevisionChanged is a predicate that is `true` if the passed lease object is a snapshot lease and if the lease -// object's holderIdentity is updated. -func SnapshotRevisionChanged() predicate.Predicate { - isSnapshotLease := func(obj client.Object) bool { - lease, ok := obj.(*coordinationv1.Lease) - if !ok { - return false - } - - return strings.HasSuffix(lease.Name, "full-snap") || strings.HasSuffix(lease.Name, "delta-snap") - } - - holderIdentityChange := func(objOld, objNew client.Object) bool { - leaseOld, ok := objOld.(*coordinationv1.Lease) - if !ok { - return false - } - leaseNew, ok := objNew.(*coordinationv1.Lease) - if !ok { - return false - } - - return !reflect.DeepEqual(leaseOld.Spec.HolderIdentity, leaseNew.Spec.HolderIdentity) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return isSnapshotLease(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return isSnapshotLease(event.ObjectNew) && holderIdentityChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return false - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} - -// JobStatusChanged is a predicate that is `true` if the status of a job changes. -func JobStatusChanged() predicate.Predicate { - statusChange := func(objOld, objNew client.Object) bool { - jobOld, ok := objOld.(*batchv1.Job) - if !ok { - return false - } - jobNew, ok := objNew.(*batchv1.Job) - if !ok { - return false - } - return !apiequality.Semantic.DeepEqual(jobOld.Status, jobNew.Status) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return false - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return statusChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return false - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} diff --git a/controllers/predicate/predicate_suite_test.go b/controllers/predicate/predicate_suite_test.go deleted file mode 100644 index d800fe30e..000000000 --- a/controllers/predicate/predicate_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package predicate_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestPredicate(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Predicate Suite") -} diff --git a/controllers/predicate/predicate_test.go b/controllers/predicate/predicate_test.go deleted file mode 100644 index f3fc75231..000000000 --- a/controllers/predicate/predicate_test.go +++ /dev/null @@ -1,658 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package predicate_test - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/controllers/predicate" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -var _ = Describe("Druid Predicate", func() { - var ( - obj, oldObj client.Object - - createEvent event.CreateEvent - updateEvent event.UpdateEvent - deleteEvent event.DeleteEvent - genericEvent event.GenericEvent - ) - - JustBeforeEach(func() { - createEvent = event.CreateEvent{ - Object: obj, - } - updateEvent = event.UpdateEvent{ - ObjectOld: oldObj, - ObjectNew: obj, - } - deleteEvent = event.DeleteEvent{ - Object: obj, - } - genericEvent = event.GenericEvent{ - Object: obj, - } - }) - - Describe("#StatefulSet", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = StatefulSetStatusChange() - }) - - Context("when status matches", func() { - BeforeEach(func() { - obj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 1, - }, - } - oldObj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 1, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when status differs", func() { - BeforeEach(func() { - obj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 2, - }, - } - oldObj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 1, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#Lease", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = SnapshotRevisionChanged() - }) - - Context("when holder identity is nil for delta snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity matches for delta snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity differs for delta snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("5"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity is nil for full snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity matches for full snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity differs for full snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("5"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity differs for any other leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("5"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - }) - - Describe("#Job", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = JobStatusChanged() - }) - - Context("when status matches", func() { - BeforeEach(func() { - now := metav1.Now() - obj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: &now, - }, - } - oldObj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: &now, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when status differs", func() { - BeforeEach(func() { - now := metav1.Now() - obj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: nil, - }, - } - oldObj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: &now, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - }) - - Describe("#LastOperationNotSuccessful", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = LastOperationNotSuccessful() - }) - - Context("when last error is not set", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - LastError: nil, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when last error is set", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - LastError: pointer.String("foo error"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#HasOperationAnnotation", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = HasOperationAnnotation() - }) - - Context("when has no operation annotation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when has operation annotation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#OR", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = predicate.Or( - HasOperationAnnotation(), - LastOperationNotSuccessful(), - ) - }) - - Context("when has neither operation annotation nor last error", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when has operation annotation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when has last error", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - }, - Status: druidv1alpha1.EtcdStatus{ - LastError: pointer.String("error"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when has both operation annotation and last error", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - }, - Status: druidv1alpha1.EtcdStatus{ - LastError: pointer.String("error"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#EtcdReconciliationFinished", func() { - var pred predicate.Predicate - - BeforeEach(func() { - pred = EtcdReconciliationFinished(false) - }) - - Context("when etcd has no reconcile operation annotation and observedGeneration is not present", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - Generation: 2, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has no reconcile operation annotation and observedGeneration is not equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(1), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has no reconcile operation annotation and observedGeneration is equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(2), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when etcd has reconcile operation annotation and observedGeneration is not present", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - Generation: 1, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has reconcile operation annotation and observedGeneration is not equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(1), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has reconcile operation annotation and observedGeneration is equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(2), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - }) -}) diff --git a/controllers/secret/config.go b/controllers/secret/config.go deleted file mode 100644 index ca9e207fd..000000000 --- a/controllers/secret/config.go +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "github.com/gardener/etcd-druid/controllers/utils" - - flag "github.com/spf13/pflag" -) - -const ( - workersFlagName = "secret-workers" - - defaultWorkers = 10 -) - -// Config defines the configuration for the Secret Controller. -type Config struct { - // Workers is the number of workers concurrently processing reconciliation requests. - Workers int -} - -// InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, - "Number of worker threads for the secrets controller.") -} - -// Validate validates the config. -func (cfg *Config) Validate() error { - return utils.MustBeGreaterThan(workersFlagName, 0, cfg.Workers) -} diff --git a/controllers/secret/reconciler.go b/controllers/secret/reconciler.go deleted file mode 100644 index 454479223..000000000 --- a/controllers/secret/reconciler.go +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/util/sets" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -// Reconciler reconciles secrets referenced in Etcd objects. -type Reconciler struct { - client.Client - Config *Config - logger logr.Logger -} - -// NewReconciler creates a new reconciler for Secret. -func NewReconciler(mgr manager.Manager, config *Config) *Reconciler { - return &Reconciler{ - Client: mgr.GetClient(), - Config: config, - logger: log.Log.WithName("secret-controller"), - } -} - -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update;patch - -// Reconcile reconciles the secret. -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - secret := &corev1.Secret{} - if err := r.Get(ctx, req.NamespacedName, secret); err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - logger := r.logger.WithValues("secret", client.ObjectKeyFromObject(secret)) - - etcdList := &druidv1alpha1.EtcdList{} - if err := r.Client.List(ctx, etcdList, client.InNamespace(secret.Namespace)); err != nil { - return ctrl.Result{}, err - } - - if needed, etcd := isFinalizerNeeded(secret.Name, etcdList); needed { - logger.Info("Adding finalizer for secret since it is referenced by etcd resource", - "secretNamespace", secret.Namespace, "secretName", secret.Name, "etcdNamespace", etcd.Namespace, "etcdName", etcd.Name) - return ctrl.Result{}, addFinalizer(ctx, logger, r.Client, secret) - } - return ctrl.Result{}, removeFinalizer(ctx, logger, r.Client, secret) -} - -func isFinalizerNeeded(secretName string, etcdList *druidv1alpha1.EtcdList) (bool, *druidv1alpha1.Etcd) { - for _, etcd := range etcdList.Items { - if etcd.Spec.Etcd.ClientUrlTLS != nil && - (etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name == secretName || - etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name == secretName || - etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name == secretName) { - return true, &etcd - } - - if etcd.Spec.Etcd.PeerUrlTLS != nil && - (etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name == secretName || - etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name == secretName) { // Currently, no client certificate for peer url is used in ETCD cluster - return true, &etcd - } - - if etcd.Spec.Backup.Store != nil && - etcd.Spec.Backup.Store.SecretRef != nil && - etcd.Spec.Backup.Store.SecretRef.Name == secretName { - return true, &etcd - } - } - - return false, nil -} - -func addFinalizer(ctx context.Context, logger logr.Logger, k8sClient client.Client, secret *corev1.Secret) error { - if finalizers := sets.NewString(secret.Finalizers...); finalizers.Has(common.FinalizerName) { - return nil - } - logger.Info("Adding finalizer", "namespace", secret.Namespace, "name", secret.Name, "finalizerName", common.FinalizerName) - return client.IgnoreNotFound(controllerutils.AddFinalizers(ctx, k8sClient, secret, common.FinalizerName)) -} - -func removeFinalizer(ctx context.Context, logger logr.Logger, k8sClient client.Client, secret *corev1.Secret) error { - if finalizers := sets.NewString(secret.Finalizers...); !finalizers.Has(common.FinalizerName) { - return nil - } - logger.Info("Removing finalizer", "namespace", secret.Namespace, "name", secret.Name, "finalizerName", common.FinalizerName) - return client.IgnoreNotFound(controllerutils.RemoveFinalizers(ctx, k8sClient, secret, common.FinalizerName)) -} diff --git a/controllers/secret/reconciler_test.go b/controllers/secret/reconciler_test.go deleted file mode 100644 index 15be4b6e5..000000000 --- a/controllers/secret/reconciler_test.go +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "context" - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("SecretController", func() { - var ( - testSecretName = "test-secret" - testNamespace = "test-namespace" - ) - - Describe("#isFinalizerNeeded", func() { - var ( - etcdList druidv1alpha1.EtcdList - ) - - BeforeEach(func() { - etcdList = druidv1alpha1.EtcdList{} - for i := 0; i < 3; i++ { - etcdList.Items = append(etcdList.Items, druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("etcd-%d", i), - Namespace: testNamespace, - }, - }) - } - }) - - It("should return false if secret is not referred to by any Etcd object", func() { - isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) - Expect(isFinalizerNeeded).To(BeFalse()) - }) - - It("should return true if secret is referred to in an Etcd object's Spec.Etcd.ClientUrlTLS section", func() { - etcdList.Items[0].Spec.Etcd.ClientUrlTLS = &druidv1alpha1.TLSConfig{ - ServerTLSSecretRef: v1.SecretReference{ - Name: testSecretName, - Namespace: testNamespace, - }, - } - isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) - Expect(isFinalizerNeeded).To(BeTrue()) - }) - - It("should return true if secret is referred to in an Etcd object's Spec.Etcd.PeerUrlTLS section", func() { - etcdList.Items[1].Spec.Etcd.PeerUrlTLS = &druidv1alpha1.TLSConfig{ - ServerTLSSecretRef: v1.SecretReference{ - Name: testSecretName, - Namespace: testNamespace, - }, - } - isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) - Expect(isFinalizerNeeded).To(BeTrue()) - }) - - It("should return true if secret is referred to in an Etcd object's Spec.Backup.Store section", func() { - etcdList.Items[2].Spec.Backup.Store = &druidv1alpha1.StoreSpec{ - SecretRef: &v1.SecretReference{ - Name: testSecretName, - Namespace: testNamespace, - }, - } - isFinalizerNeeded, _ := isFinalizerNeeded(testSecretName, &etcdList) - Expect(isFinalizerNeeded).To(BeTrue()) - }) - }) - - Describe("#addFinalizer", func() { - var ( - secret *corev1.Secret - ctx context.Context - logger = logr.Discard() - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - ) - - Context("test #addFinalizer on existing secret", func() { - BeforeEach(func() { - ctx = context.Background() - secret = ensureSecretCreation(ctx, testSecretName, testNamespace, fakeClient) - }) - - AfterEach(func() { - ensureSecretRemoval(ctx, testSecretName, testNamespace, fakeClient) - }) - - It("should return nil if secret already has finalizer", func() { - Expect(controllerutils.AddFinalizers(ctx, fakeClient, secret, common.FinalizerName)).To(Succeed()) - Expect(addFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) - }) - - It("should add finalizer on secret if finalizer does not exist", func() { - Expect(addFinalizer(ctx, logger, fakeClient, secret)).To(Succeed()) - finalizers := sets.NewString(secret.Finalizers...) - Expect(finalizers.Has(common.FinalizerName)).To(BeTrue()) - }) - }) - - Context("test #addFinalizer when secret does not exist", func() { - BeforeEach(func() { - ctx = context.Background() - secret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "non-existent", - Namespace: testNamespace, - // resource version is required as OptimisticLock is used for the merge patch - // operation used by controllerutils.AddFinalizers() - ResourceVersion: "42", - }, - } - }) - - It("should not return an error if the secret is not found", func() { - Expect(addFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) - }) - }) - }) - - Describe("#removeFinalizer", func() { - var ( - secret *corev1.Secret - ctx context.Context - logger = logr.Discard() - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - ) - - Context("test remove finalizer on an existing secret", func() { - BeforeEach(func() { - ctx = context.Background() - secret = ensureSecretCreation(ctx, testSecretName, testNamespace, fakeClient) - }) - - AfterEach(func() { - ensureSecretRemoval(ctx, testSecretName, testNamespace, fakeClient) - }) - - It("should return nil if secret does not have finalizer", func() { - Expect(controllerutils.RemoveAllFinalizers(ctx, fakeClient, secret)).Should(Succeed()) - Expect(removeFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) - }) - - It("should remove finalizer from secret if finalizer exists", func() { - Expect(removeFinalizer(ctx, logger, fakeClient, secret)).To(Succeed()) - finalizers := sets.NewString(secret.Finalizers...) - Expect(finalizers.Has(common.FinalizerName)).To(BeFalse()) - }) - }) - - Context("test remove finalizer on a non-existing secret", func() { - BeforeEach(func() { - ctx = context.Background() - secret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "non-existent", - Namespace: testNamespace, - }, - } - }) - - It("should not return an error if the secret is not found", func() { - Expect(removeFinalizer(ctx, logger, fakeClient, secret)).To(BeNil()) - }) - }) - - }) -}) - -func ensureSecretCreation(ctx context.Context, name, namespace string, fakeClient client.WithWatch) *corev1.Secret { - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Finalizers: []string{ - common.FinalizerName, - }, - }, - } - - By("Create Secret") - Expect(fakeClient.Create(ctx, secret)).To(Succeed()) - - By("Ensure Secret is created") - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(secret), secret) - }).Should(Succeed()) - - return secret -} - -func ensureSecretRemoval(ctx context.Context, name, namespace string, fakeClient client.WithWatch) { - secret := &corev1.Secret{} - if err := fakeClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret); err != nil { - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - return - } - - By("Remove any existing finalizers on Secret") - Expect(controllerutils.RemoveAllFinalizers(ctx, fakeClient, secret)).To(Succeed()) - - By("Delete Secret") - Expect(fakeClient.Delete(ctx, secret)).To(Succeed()) - - By("Ensure Secret is deleted") - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(secret), secret) - }).Should(BeNotFoundError()) -} diff --git a/controllers/secret/register.go b/controllers/secret/register.go deleted file mode 100644 index 15a384a40..000000000 --- a/controllers/secret/register.go +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidmapper "github.com/gardener/etcd-druid/pkg/mapper" - - "github.com/gardener/gardener/pkg/controllerutils/mapper" - corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -const controllerName = "secret-controller" - -// RegisterWithManager registers the Secret Controller with the given controller manager. -func (r *Reconciler) RegisterWithManager(ctx context.Context, mgr ctrl.Manager) error { - c, err := ctrl. - NewControllerManagedBy(mgr). - Named(controllerName). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.Config.Workers, - }). - For(&corev1.Secret{}). - Build(r) - if err != nil { - return err - } - - return c.Watch( - source.Kind(mgr.GetCache(), &druidv1alpha1.Etcd{}), - mapper.EnqueueRequestsFrom(ctx, mgr.GetCache(), druidmapper.EtcdToSecret(), mapper.UpdateWithOldAndNew, c.GetLogger()), - ) -} diff --git a/controllers/secret/secret_suite_test.go b/controllers/secret/secret_suite_test.go deleted file mode 100644 index 8286b0674..000000000 --- a/controllers/secret/secret_suite_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package secret - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestSecretController(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs( - t, - "Secret Controller Suite", - ) -} diff --git a/controllers/utils/reconciler.go b/controllers/utils/reconciler.go deleted file mode 100644 index 07b56f077..000000000 --- a/controllers/utils/reconciler.go +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/gardener/pkg/utils/imagevector" -) - -// CreateImageVector creates an image vector from the default images.yaml file or the images-wrapper.yaml file. -func CreateImageVector() (imagevector.ImageVector, error) { - imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(common.DefaultImageVectorFilePath) - if err != nil { - return nil, err - } - return imageVector, nil -} diff --git a/controllers/utils/utils_suite_test.go b/controllers/utils/utils_suite_test.go deleted file mode 100644 index a3123f5d4..000000000 --- a/controllers/utils/utils_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestControllerUtils(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Controller Utils suite") -} diff --git a/controllers/utils/validator.go b/controllers/utils/validator.go deleted file mode 100644 index a189de47d..000000000 --- a/controllers/utils/validator.go +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "fmt" - - "golang.org/x/exp/constraints" -) - -// ShouldBeOneOfAllowedValues checks if value is amongst the allowedValues. If it is not then an error is returned else nil is returned. -// Type is constrained by comparable forcing the consumers to only use concrete types that can be compared using the == or != operators. -func ShouldBeOneOfAllowedValues[E comparable](key string, allowedValues []E, value E) error { - for _, av := range allowedValues { - if av == value { - return nil - } - } - return fmt.Errorf("unsupported value %v provided for %s. allowed values are: %v", value, key, allowedValues) -} - -// MustBeGreaterThan checks if the value is greater than the lowerBound. If it is not then an error is returned else nil is returned. -// Type is constrained by constraints.Ordered which enforces the consumers to use concrete types that can be compared using >, >=, <, <= operators. -func MustBeGreaterThan[E constraints.Ordered](key string, lowerBound, value E) error { - if value <= lowerBound { - return fmt.Errorf("%s should have a value greater than %v. value provided is %v", key, lowerBound, value) - } - return nil -} - -// MustBeGreaterThanOrEqualTo checks if the value is greater than or equal to the lowerBound. If it is not then an error is returned else nil is returned. -// Type is constrained by constraints.Ordered which enforces the consumers to use concrete types that can be compared using >, >=, <, <= operators. -func MustBeGreaterThanOrEqualTo[E constraints.Ordered](key string, lowerBound, value E) error { - if value < lowerBound { - return fmt.Errorf("%s should have a value greater or equal to %v. value provided is %v", key, lowerBound, value) - } - return nil -} diff --git a/controllers/utils/validator_test.go b/controllers/utils/validator_test.go deleted file mode 100644 index 64719fc5d..000000000 --- a/controllers/utils/validator_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Validator Tests", func() { - - DescribeTable("#ShouldBeOneOfAllowedValues tests", - func(key string, allowedValues []string, valueToCheck string, expectError bool) { - err := ShouldBeOneOfAllowedValues(key, allowedValues, valueToCheck) - Expect(err != nil).To(Equal(expectError)) - }, - Entry("value is not present in the allowed values slice", "locknames", []string{"configmaps"}, "leases", true), - Entry("values is present in the allowed values slice", "locknames", []string{"configmaps", "leases", "endpoints"}, "leases", false), - ) - - DescribeTable("#MustBeGreaterThan", - func(key string, lowerBound int, value int, expectError bool) { - err := MustBeGreaterThan(key, lowerBound, value) - Expect(err != nil).To(Equal(expectError)) - }, - Entry("value is not greater than the lower bound", "workers", 0, -1, true), - Entry("value is greater than the lower bound", "durationSeconds", 10, 20, false), - Entry("value is equal to the lower bound", "threshold", 10, 10, true), - ) - - DescribeTable("#MustBeGreaterThanOrEqualTo", - func(key string, lowerBound int, value int, expectError bool) { - err := MustBeGreaterThanOrEqualTo(key, lowerBound, value) - Expect(err != nil).To(Equal(expectError)) - }, - Entry("value is not greater than the lower bound", "workers", 0, -1, true), - Entry("value is greater than the lower bound", "durationSeconds", 10, 20, false), - Entry("value is equal to the lower bound", "threshold", 10, 10, false), - ) - -}) diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 4d82477d5..474a3a086 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -7,7 +7,7 @@ package compaction import ( "time" - "github.com/gardener/etcd-druid/controllers/utils" + "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" flag "github.com/spf13/pflag" diff --git a/internal/controller/compaction/register.go b/internal/controller/compaction/register.go index f8dd65db6..f83e58bbd 100644 --- a/internal/controller/compaction/register.go +++ b/internal/controller/compaction/register.go @@ -6,7 +6,7 @@ package compaction import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/controllers/predicate" + druidpredicates "github.com/gardener/etcd-druid/internal/controller/predicate" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" diff --git a/internal/controller/etcdcopybackupstask/config.go b/internal/controller/etcdcopybackupstask/config.go index 9bde160db..66ff9edfd 100644 --- a/internal/controller/etcdcopybackupstask/config.go +++ b/internal/controller/etcdcopybackupstask/config.go @@ -5,8 +5,8 @@ package etcdcopybackupstask import ( - "github.com/gardener/etcd-druid/controllers/utils" - "github.com/gardener/etcd-druid/pkg/features" + "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/features" flag "github.com/spf13/pflag" "k8s.io/component-base/featuregate" diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 1f21546c3..20b3f3063 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -10,8 +10,8 @@ import ( "strconv" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - ctrlutils "github.com/gardener/etcd-druid/controllers/utils" "github.com/gardener/etcd-druid/internal/common" + ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/utils" diff --git a/internal/controller/predicate/predicate.go b/internal/controller/predicate/predicate.go index 58554ddbc..d82b885ad 100644 --- a/internal/controller/predicate/predicate.go +++ b/internal/controller/predicate/predicate.go @@ -8,108 +8,14 @@ import ( "reflect" "strings" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ) -func hasOperationAnnotation(obj client.Object) bool { - return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile -} - -// HasOperationAnnotation is a predicate for the operation annotation. -func HasOperationAnnotation() predicate.Predicate { - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return hasOperationAnnotation(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return hasOperationAnnotation(event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return hasOperationAnnotation(event.Object) - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return true - }, - } -} - -// LastOperationNotSuccessful is a predicate for unsuccessful last operations for creation events. -func LastOperationNotSuccessful() predicate.Predicate { - operationNotSucceeded := func(obj runtime.Object) bool { - etcd, ok := obj.(*druidv1alpha1.Etcd) - if !ok { - return false - } - if etcd.Status.LastError != nil { - return true - } - return false - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return operationNotSucceeded(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return operationNotSucceeded(event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return operationNotSucceeded(event.Object) - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return operationNotSucceeded(event.Object) - }, - } -} - -// EtcdReconciliationFinished is a predicate to use for etcd resources whose reconciliation has finished. -func EtcdReconciliationFinished(ignoreOperationAnnotation bool) predicate.Predicate { - reconciliationFinished := func(obj client.Object) bool { - etcd, ok := obj.(*druidv1alpha1.Etcd) - if !ok { - return false - } - - if etcd.Status.ObservedGeneration == nil { - return false - } - - condition := *etcd.Status.ObservedGeneration == etcd.Generation - - if !ignoreOperationAnnotation { - condition = condition && !hasOperationAnnotation(etcd) - } - - return condition - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return reconciliationFinished(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return reconciliationFinished(event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return reconciliationFinished(event.Object) - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} - // SnapshotRevisionChanged is a predicate that is `true` if the passed lease object is a snapshot lease and if the lease // object's holderIdentity is updated. func SnapshotRevisionChanged() predicate.Predicate { @@ -180,83 +86,3 @@ func JobStatusChanged() predicate.Predicate { }, } } - -// StatefulSetSpecChange is a predicate for status changes of `StatefulSet` resources. -func StatefulSetSpecChange() predicate.Predicate { - specChange := func(objOld, objNew client.Object) bool { - stsOld, ok := objOld.(*appsv1.StatefulSet) - if !ok { - return false - } - stsNew, ok := objNew.(*appsv1.StatefulSet) - if !ok { - return false - } - return !apiequality.Semantic.DeepEqual(stsOld.Spec, stsNew.Spec) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return true - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return specChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return true - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return true - }, - } -} - -func StatefulSetContainerResourcesChange() predicate.Predicate { - containerResourcesChange := func(objOld, objNew client.Object) bool { - stsOld, ok := objOld.(*appsv1.StatefulSet) - if !ok { - return false - } - stsNew, ok := objNew.(*appsv1.StatefulSet) - if !ok { - return false - } - - containersOld := make(map[string]*corev1.Container) - containersNew := make(map[string]*corev1.Container) - - if len(stsOld.Spec.Template.Spec.Containers) != len(stsNew.Spec.Template.Spec.Containers) { - return true - } - for _, c := range stsOld.Spec.Template.Spec.Containers { - containersOld[c.Name] = &c - } - for _, cNew := range stsNew.Spec.Template.Spec.Containers { - containersNew[cNew.Name] = &cNew - cOld, ok := containersOld[cNew.Name] - if !ok { - return true - } - if !apiequality.Semantic.DeepEqual(cNew.Resources, cOld.Resources) { - return true - } - } - - return false - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return false - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return containerResourcesChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return false - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} diff --git a/internal/controller/predicate/predicate_test.go b/internal/controller/predicate/predicate_test.go index f3fc75231..aa2e08f29 100644 --- a/internal/controller/predicate/predicate_test.go +++ b/internal/controller/predicate/predicate_test.go @@ -5,14 +5,11 @@ package predicate_test import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/controllers/predicate" + . "github.com/gardener/etcd-druid/internal/controller/predicate" . "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,58 +46,6 @@ var _ = Describe("Druid Predicate", func() { } }) - Describe("#StatefulSet", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = StatefulSetStatusChange() - }) - - Context("when status matches", func() { - BeforeEach(func() { - obj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 1, - }, - } - oldObj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 1, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when status differs", func() { - BeforeEach(func() { - obj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 2, - }, - } - oldObj = &appsv1.StatefulSet{ - Status: appsv1.StatefulSetStatus{ - Replicas: 1, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - Describe("#Lease", func() { var pred predicate.Predicate @@ -346,313 +291,4 @@ var _ = Describe("Druid Predicate", func() { }) }) }) - - Describe("#LastOperationNotSuccessful", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = LastOperationNotSuccessful() - }) - - Context("when last error is not set", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - LastError: nil, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when last error is set", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - LastError: pointer.String("foo error"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#HasOperationAnnotation", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = HasOperationAnnotation() - }) - - Context("when has no operation annotation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when has operation annotation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#OR", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = predicate.Or( - HasOperationAnnotation(), - LastOperationNotSuccessful(), - ) - }) - - Context("when has neither operation annotation nor last error", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when has operation annotation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when has last error", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - }, - Status: druidv1alpha1.EtcdStatus{ - LastError: pointer.String("error"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when has both operation annotation and last error", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - }, - Status: druidv1alpha1.EtcdStatus{ - LastError: pointer.String("error"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - }) - - Describe("#EtcdReconciliationFinished", func() { - var pred predicate.Predicate - - BeforeEach(func() { - pred = EtcdReconciliationFinished(false) - }) - - Context("when etcd has no reconcile operation annotation and observedGeneration is not present", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - Generation: 2, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has no reconcile operation annotation and observedGeneration is not equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(1), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has no reconcile operation annotation and observedGeneration is equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: make(map[string]string), - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(2), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeTrue()) - }) - }) - - Context("when etcd has reconcile operation annotation and observedGeneration is not present", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - Generation: 1, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has reconcile operation annotation and observedGeneration is not equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(1), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when etcd has reconcile operation annotation and observedGeneration is equal to generation", func() { - BeforeEach(func() { - obj = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - }, - Generation: 2, - }, - Status: druidv1alpha1.EtcdStatus{ - ObservedGeneration: pointer.Int64(2), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - }) }) diff --git a/internal/controller/secret/config.go b/internal/controller/secret/config.go index ca9e207fd..c45c95631 100644 --- a/internal/controller/secret/config.go +++ b/internal/controller/secret/config.go @@ -5,7 +5,7 @@ package secret import ( - "github.com/gardener/etcd-druid/controllers/utils" + "github.com/gardener/etcd-druid/internal/controller/utils" flag "github.com/spf13/pflag" ) diff --git a/internal/controller/secret/reconciler.go b/internal/controller/secret/reconciler.go index 454479223..81eda1902 100644 --- a/internal/controller/secret/reconciler.go +++ b/internal/controller/secret/reconciler.go @@ -8,7 +8,7 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" diff --git a/internal/controller/secret/register.go b/internal/controller/secret/register.go index 44ab3824c..b15d1a92f 100644 --- a/internal/controller/secret/register.go +++ b/internal/controller/secret/register.go @@ -8,7 +8,7 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidmapper "github.com/gardener/etcd-druid/pkg/mapper" + druidmapper "github.com/gardener/etcd-druid/internal/mapper" "github.com/gardener/gardener/pkg/controllerutils/mapper" corev1 "k8s.io/api/core/v1" diff --git a/internal/health/condition/builder.go b/internal/health/condition/builder.go index f979112af..f7c48b13b 100644 --- a/internal/health/condition/builder.go +++ b/internal/health/condition/builder.go @@ -13,7 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// skipMergeConditions contain the list of conditions we dont want to add to the list if not recalculated +// skipMergeConditions contain the list of conditions we don't want to add to the list if not recalculated var skipMergeConditions = map[druidv1alpha1.ConditionType]struct{}{ druidv1alpha1.ConditionTypeReady: {}, druidv1alpha1.ConditionTypeAllMembersReady: {}, diff --git a/internal/health/condition/builder_test.go b/internal/health/condition/builder_test.go index 17f5a6da0..5f197ca31 100644 --- a/internal/health/condition/builder_test.go +++ b/internal/health/condition/builder_test.go @@ -13,7 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" + . "github.com/gardener/etcd-druid/internal/health/condition" ) var _ = Describe("Builder", func() { diff --git a/internal/health/condition/check_backup_ready_test.go b/internal/health/condition/check_backup_ready_test.go index 4216d4361..06a9a12f9 100644 --- a/internal/health/condition/check_backup_ready_test.go +++ b/internal/health/condition/check_backup_ready_test.go @@ -10,7 +10,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" . "github.com/gardener/etcd-druid/internal/health/condition" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" + mockclient "github.com/gardener/etcd-druid/internal/mock/controller-runtime/client" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" diff --git a/internal/mapper/etcd_to_secret_test.go b/internal/mapper/etcd_to_secret_test.go index ed7526817..f605c738b 100644 --- a/internal/mapper/etcd_to_secret_test.go +++ b/internal/mapper/etcd_to_secret_test.go @@ -8,7 +8,7 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/mapper" + . "github.com/gardener/etcd-druid/internal/mapper" "github.com/go-logr/logr" "github.com/gardener/gardener/pkg/controllerutils/mapper" diff --git a/internal/mapper/statefulset_to_etcd.go b/internal/mapper/statefulset_to_etcd.go deleted file mode 100644 index a174a39f7..000000000 --- a/internal/mapper/statefulset_to_etcd.go +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils/mapper" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/gardener/etcd-druid/pkg/common" -) - -type statefulSetToEtcdMapper struct { - ctx context.Context - cl client.Client -} - -func (m *statefulSetToEtcdMapper) Map(_ context.Context, _ logr.Logger, _ client.Reader, obj client.Object) []reconcile.Request { - sts, ok := obj.(*appsv1.StatefulSet) - if !ok { - return nil - } - - ownerKey, ok := sts.Annotations[common.GardenerOwnedBy] - if !ok { - return nil - } - - name, namespace, err := cache.SplitMetaNamespaceKey(ownerKey) - if err != nil { - return nil - } - - etcd := &druidv1alpha1.Etcd{} - if err := m.cl.Get(m.ctx, kutil.Key(name, namespace), etcd); err != nil { - return nil - } - - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Name, - }, - }, - } -} - -// StatefulSetToEtcd returns a mapper that returns a request for the Etcd resource -// that owns the StatefulSet for which an event happened. -func StatefulSetToEtcd(ctx context.Context, cl client.Client) mapper.Mapper { - return &statefulSetToEtcdMapper{ - ctx: ctx, - cl: cl, - } -} diff --git a/internal/mapper/statefulset_to_etcd_test.go b/internal/mapper/statefulset_to_etcd_test.go deleted file mode 100644 index f3bff090e..000000000 --- a/internal/mapper/statefulset_to_etcd_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper_test - -import ( - "context" - "fmt" - - "github.com/gardener/gardener/pkg/controllerutils/mapper" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - . "github.com/gardener/etcd-druid/pkg/mapper" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" -) - -var _ = Describe("Druid Mapper", func() { - var ( - ctx = context.Background() - ctrl *gomock.Controller - c *mockclient.MockClient - logger logr.Logger - - name, namespace, key string - statefulset *appsv1.StatefulSet - mapper mapper.Mapper - ) - - BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) - c = mockclient.NewMockClient(ctrl) - logger = log.Log.WithName("Test") - - name = "etcd-test" - namespace = "test" - key = fmt.Sprintf("%s/%s", namespace, name) - statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - mapper = StatefulSetToEtcd(ctx, c) - }) - - AfterEach(func() { - ctrl.Finish() - }) - - Describe("#StatefulSetToEtcd", func() { - It("should find related Etcd object", func() { - etcd := &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - - c.EXPECT().Get(ctx, kutil.Key(namespace, name), gomock.AssignableToTypeOf(etcd), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, obj *druidv1alpha1.Etcd, _ ...client.GetOption) error { - *obj = *etcd - return nil - }, - ) - - kutil.SetMetaDataAnnotation(statefulset, common.GardenerOwnedBy, key) - - etcds := mapper.Map(ctx, logger, nil, statefulset) - - Expect(etcds).To(ConsistOf( - reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: namespace, - Name: name, - }, - }, - )) - }) - - It("should not find related Etcd object because an error occurred during retrieval", func() { - c.EXPECT().Get(ctx, kutil.Key(namespace, name), gomock.AssignableToTypeOf(&druidv1alpha1.Etcd{})).Return(fmt.Errorf("foo error")) - - kutil.SetMetaDataAnnotation(statefulset, common.GardenerOwnedBy, key) - - etcds := mapper.Map(ctx, logger, nil, statefulset) - - Expect(etcds).To(BeEmpty()) - }) - - It("should not find related Etcd object because owner annotation is not present", func() { - etcds := mapper.Map(ctx, logger, nil, statefulset) - - Expect(etcds).To(BeEmpty()) - }) - - It("should not find related Etcd object because map is called with wrong object", func() { - etcds := mapper.Map(ctx, logger, nil, nil) - - Expect(etcds).To(BeEmpty()) - }) - }) -}) diff --git a/main.go b/main.go index b179f7643..575a669f3 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/gardener/etcd-druid/internal/controller" - "github.com/gardener/etcd-druid/pkg/version" + "github.com/gardener/etcd-druid/internal/version" flag "github.com/spf13/pflag" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" diff --git a/pkg/client/kubernetes/types.go b/pkg/client/kubernetes/types.go deleted file mode 100644 index 12237375d..000000000 --- a/pkg/client/kubernetes/types.go +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package kubernetes - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - schemev1 "k8s.io/client-go/kubernetes/scheme" -) - -// Scheme is the kubernetes runtime scheme -var Scheme = runtime.NewScheme() - -func init() { - localSchemeBuilder := runtime.NewSchemeBuilder( - druidv1alpha1.AddToScheme, - schemev1.AddToScheme, - ) - - utilruntime.Must(localSchemeBuilder.AddToScheme(Scheme)) -} diff --git a/pkg/common/constants.go b/pkg/common/constants.go deleted file mode 100644 index 5a76b5669..000000000 --- a/pkg/common/constants.go +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package common - -const ( - // Etcd is the key for the etcd image in the image vector. - Etcd = "etcd" - // BackupRestore is the key for the etcd-backup-restore image in the image vector. - BackupRestore = "etcd-backup-restore" - // EtcdWrapper is the key for the etcd image in the image vector. - EtcdWrapper = "etcd-wrapper" - // BackupRestoreDistroless is the key for the etcd-backup-restore image in the image vector. - BackupRestoreDistroless = "etcd-backup-restore-distroless" - // Alpine is the key for the alpine image in the image vector. - Alpine = "alpine" - // DefaultImageVectorFilePath is the path to the default image vector file. - DefaultImageVectorFilePath = "charts/images.yaml" - // GardenerOwnedBy is a constant for an annotation on a resource that describes the owner resource. - GardenerOwnedBy = "gardener.cloud/owned-by" - // GardenerOwnerType is a constant for an annotation on a resource that describes the type of owner resource. - GardenerOwnerType = "gardener.cloud/owner-type" - // FinalizerName is the name of the etcd finalizer. - FinalizerName = "druid.gardener.cloud/etcd-druid" - - // EnvPodName is the environment variable key for the pod name. - EnvPodName = "POD_NAME" - // EnvPodNamespace is the environment variable key for the pod namespace. - EnvPodNamespace = "POD_NAMESPACE" - // EnvStorageContainer is the environment variable key for the storage container. - EnvStorageContainer = "STORAGE_CONTAINER" - // EnvSourceStorageContainer is the environment variable key for the source storage container. - EnvSourceStorageContainer = "SOURCE_STORAGE_CONTAINER" - - // EnvAWSApplicationCredentials is the environment variable key for AWS application credentials. - EnvAWSApplicationCredentials = "AWS_APPLICATION_CREDENTIALS" - // EnvAzureApplicationCredentials is the environment variable key for Azure application credentials. - EnvAzureApplicationCredentials = "AZURE_APPLICATION_CREDENTIALS" - // EnvGoogleApplicationCredentials is the environment variable key for Google application credentials. - EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS" - // EnvGoogleStorageAPIEndpoint is the environment variable key for Google storage API endpoint override. - EnvGoogleStorageAPIEndpoint = "GOOGLE_STORAGE_API_ENDPOINT" - // EnvOpenstackApplicationCredentials is the environment variable key for OpenStack application credentials. - EnvOpenstackApplicationCredentials = "OPENSTACK_APPLICATION_CREDENTIALS" - // EnvAlicloudApplicationCredentials is the environment variable key for Alicloud application credentials. - EnvAlicloudApplicationCredentials = "ALICLOUD_APPLICATION_CREDENTIALS" - // EnvOpenshiftApplicationCredentials is the environment variable key for OpenShift application credentials. - EnvOpenshiftApplicationCredentials = "OPENSHIFT_APPLICATION_CREDENTIALS" - // EnvECSEndpoint is the environment variable key for Dell ECS endpoint. - EnvECSEndpoint = "ECS_ENDPOINT" - // EnvECSAccessKeyID is the environment variable key for Dell ECS access key ID. - EnvECSAccessKeyID = "ECS_ACCESS_KEY_ID" - // EnvECSSecretAccessKey is the environment variable key for Dell ECS secret access key. - EnvECSSecretAccessKey = "ECS_SECRET_ACCESS_KEY" -) diff --git a/pkg/component/etcd/configmap/configmap.go b/pkg/component/etcd/configmap/configmap.go deleted file mode 100644 index ec5e9e766..000000000 --- a/pkg/component/etcd/configmap/configmap.go +++ /dev/null @@ -1,184 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package configmap - -import ( - "context" - "encoding/json" - "fmt" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - "gopkg.in/yaml.v2" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils" -) - -type component struct { - client client.Client - namespace string - - values *Values -} - -// New creates a new configmap deployer instance. -func New(c client.Client, namespace string, values *Values) gardenercomponent.Deployer { - return &component{ - client: c, - namespace: namespace, - values: values, - } -} - -func (c *component) Deploy(ctx context.Context) error { - cm := c.emptyConfigmap() - return c.syncConfigmap(ctx, cm) -} - -func (c *component) Destroy(ctx context.Context) error { - configMap := c.emptyConfigmap() - return c.deleteConfigmap(ctx, configMap) -} - -type etcdTLSTarget string - -const ( - clientTLS etcdTLSTarget = "client" - peerTLS etcdTLSTarget = "peer" - defaultQuota = int64(8 * 1024 * 1024 * 1024) // 8Gi - defaultSnapshotCount = 75000 // for more information refer to https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention - defaultInitialClusterState = "new" - defaultInitialClusterToken = "etcd-cluster" -) - -func (c *component) syncConfigmap(ctx context.Context, cm *corev1.ConfigMap) error { - peerScheme, peerSecurity := getSchemeAndSecurityConfig(c.values.PeerUrlTLS, peerTLS) - clientScheme, clientSecurity := getSchemeAndSecurityConfig(c.values.ClientUrlTLS, clientTLS) - - quota := defaultQuota - if c.values.Quota != nil { - quota = c.values.Quota.Value() - } - - autoCompactionMode := string(druidv1alpha1.Periodic) - if c.values.AutoCompactionMode != nil { - autoCompactionMode = string(*c.values.AutoCompactionMode) - } - - etcdConfig := &etcdConfig{ - Name: fmt.Sprintf("etcd-%s", c.values.EtcdUID[:6]), - DataDir: "/var/etcd/data/new.etcd", - Metrics: string(druidv1alpha1.Basic), - SnapshotCount: defaultSnapshotCount, - EnableV2: false, - QuotaBackendBytes: quota, - InitialClusterToken: defaultInitialClusterToken, - InitialClusterState: defaultInitialClusterState, - InitialCluster: c.values.InitialCluster, - AutoCompactionMode: autoCompactionMode, - AutoCompactionRetention: pointer.StringDeref(c.values.AutoCompactionRetention, "30m"), - - ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, pointer.Int32Deref(c.values.ServerPort, defaultServerPort)), - ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, pointer.Int32Deref(c.values.ClientPort, defaultClientPort)), - AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, c.values.PeerServiceName, c.namespace, pointer.Int32Deref(c.values.ServerPort, defaultServerPort)), - AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, c.values.PeerServiceName, c.namespace, pointer.Int32Deref(c.values.ClientPort, defaultClientPort)), - } - - if clientSecurity != nil { - etcdConfig.ClientSecurity = *clientSecurity - } - - if peerSecurity != nil { - etcdConfig.PeerSecurity = *peerSecurity - } - - etcdConfigYaml, err := yaml.Marshal(etcdConfig) - if err != nil { - return err - } - - _, err = controllerutils.GetAndCreateOrMergePatch(ctx, c.client, cm, func() error { - cm.Name = c.values.Name - cm.Namespace = c.namespace - cm.Labels = c.values.Labels - cm.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - cm.Data = map[string]string{"etcd.conf.yaml": string(etcdConfigYaml)} - return nil - }) - if err != nil { - return err - } - - // save the checksum value of generated etcd config - jsonString, err := json.Marshal(cm.Data) - if err != nil { - return err - } - c.values.ConfigMapChecksum = utils.ComputeSHA256Hex(jsonString) - return nil -} - -func (c *component) deleteConfigmap(ctx context.Context, cm *corev1.ConfigMap) error { - return client.IgnoreNotFound(c.client.Delete(ctx, cm)) -} - -func (c *component) emptyConfigmap() *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.Name, - Namespace: c.namespace, - }, - } -} - -type etcdConfig struct { - Name string `yaml:"name"` - DataDir string `yaml:"data-dir"` - Metrics string `yaml:"metrics"` - SnapshotCount int `yaml:"snapshot-count"` - EnableV2 bool `yaml:"enable-v2"` - QuotaBackendBytes int64 `yaml:"quota-backend-bytes"` - InitialClusterToken string `yaml:"initial-cluster-token"` - InitialClusterState string `yaml:"initial-cluster-state"` - InitialCluster string `yaml:"initial-cluster"` - AutoCompactionMode string `yaml:"auto-compaction-mode"` - AutoCompactionRetention string `yaml:"auto-compaction-retention"` - ListenPeerUrls string `yaml:"listen-peer-urls"` - ListenClientUrls string `yaml:"listen-client-urls"` - AdvertisePeerUrls string `yaml:"initial-advertise-peer-urls"` - AdvertiseClientUrls string `yaml:"advertise-client-urls"` - ClientSecurity securityConfig `yaml:"client-transport-security,omitempty"` - PeerSecurity securityConfig `yaml:"peer-transport-security,omitempty"` -} - -type securityConfig struct { - CertFile string `yaml:"cert-file,omitempty"` - KeyFile string `yaml:"key-file,omitempty"` - ClientCertAuth bool `yaml:"client-cert-auth,omitempty"` - TrustedCAFile string `yaml:"trusted-ca-file,omitempty"` - AutoTLS bool `yaml:"auto-tls"` -} - -func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, tlsTarget etcdTLSTarget) (string, *securityConfig) { - if tlsConfig != nil { - tlsCASecretKey := "ca.crt" - if dataKey := tlsConfig.TLSCASecretRef.DataKey; dataKey != nil { - tlsCASecretKey = *dataKey - } - return "https", &securityConfig{ - CertFile: fmt.Sprintf("/var/etcd/ssl/%s/server/tls.crt", tlsTarget), - KeyFile: fmt.Sprintf("/var/etcd/ssl/%s/server/tls.key", tlsTarget), - ClientCertAuth: true, - TrustedCAFile: fmt.Sprintf("/var/etcd/ssl/%s/ca/%s", tlsTarget, tlsCASecretKey), - AutoTLS: false, - } - } - return "http", nil -} diff --git a/pkg/component/etcd/configmap/configmap_suite_test.go b/pkg/component/etcd/configmap/configmap_suite_test.go deleted file mode 100644 index 7ea7b49d2..000000000 --- a/pkg/component/etcd/configmap/configmap_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package configmap - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestConfigMap(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Configmap Component Suite") -} diff --git a/pkg/component/etcd/configmap/configmap_test.go b/pkg/component/etcd/configmap/configmap_test.go deleted file mode 100644 index f80492678..000000000 --- a/pkg/component/etcd/configmap/configmap_test.go +++ /dev/null @@ -1,233 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package configmap_test - -import ( - "context" - "encoding/json" - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - . "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" - "github.com/ghodss/yaml" - - "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/utils" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -const etcdConfig = "etcd.conf.yaml" - -var _ = Describe("Configmap", func() { - var ( - ctx context.Context - cl client.Client - - etcd *druidv1alpha1.Etcd - namespace string - name string - uid types.UID - - metricsLevel druidv1alpha1.MetricsLevel - quota resource.Quantity - clientPort, serverPort int32 - autoCompactionMode druidv1alpha1.CompactionMode - autoCompactionRetention string - labels map[string]string - - cm *corev1.ConfigMap - - values *Values - configmapDeployer component.Deployer - ) - - BeforeEach(func() { - ctx = context.Background() - cl = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - - name = "configmap" - namespace = "default" - uid = "a9b8c7d6e5f4" - - metricsLevel = druidv1alpha1.Basic - quota = resource.MustParse("8Gi") - clientPort = 2222 - serverPort = 3333 - autoCompactionMode = "periodic" - autoCompactionRetention = "30m" - - labels = map[string]string{ - "app": "etcd-statefulset", - "instance": "configmap", - "name": "etcd", - "role": "main", - } - - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: uid, - }, - Spec: druidv1alpha1.EtcdSpec{ - Labels: labels, - Selector: metav1.SetAsLabelSelector(labels), - Replicas: 3, - Etcd: druidv1alpha1.EtcdConfig{ - Quota: "a, - Metrics: &metricsLevel, - ClientUrlTLS: &druidv1alpha1.TLSConfig{ - ClientTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-client-tls", - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-server-cert", - }, - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "client-url-ca-etcd", - }, - }, - }, - PeerUrlTLS: &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "peer-url-ca-etcd", - }, - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "peer-url-etcd-server-tls", - }, - }, - ClientPort: &clientPort, - ServerPort: &serverPort, - }, - Common: druidv1alpha1.SharedConfig{ - AutoCompactionMode: &autoCompactionMode, - AutoCompactionRetention: &autoCompactionRetention, - }, - }, - } - - values = GenerateValues(etcd) - - cm = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("etcd-bootstrap-%s", string(values.EtcdUID[:6])), - Namespace: namespace, - }, - } - - configmapDeployer = New(cl, namespace, values) - }) - - Describe("#Deploy", func() { - Context("when configmap does not exist", func() { - It("should create the configmap successfully", func() { - Expect(configmapDeployer.Deploy(ctx)).To(Succeed()) - - cm := &corev1.ConfigMap{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), cm)).To(Succeed()) - checkConfigMap(cm, values, namespace) - - }) - }) - - Context("when configmap exists", func() { - It("should update the configmap successfully", func() { - Expect(cl.Create(ctx, cm)).To(Succeed()) - - Expect(configmapDeployer.Deploy(ctx)).To(Succeed()) - - cm := &corev1.ConfigMap{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), cm)).To(Succeed()) - checkConfigMap(cm, values, namespace) - }) - }) - }) - - Describe("#Destroy", func() { - Context("when configmap do not exist", func() { - It("should destroy successfully", func() { - Expect(configmapDeployer.Destroy(ctx)).To(Succeed()) - Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), &corev1.ConfigMap{})).To(BeNotFoundError()) - }) - }) - - Context("when configmap exist", func() { - It("should destroy successfully", func() { - Expect(cl.Create(ctx, cm)).To(Succeed()) - - Expect(configmapDeployer.Destroy(ctx)).To(Succeed()) - - Expect(cl.Get(ctx, kutil.Key(namespace, cm.Name), &corev1.ConfigMap{})).To(BeNotFoundError()) - }) - }) - }) -}) - -func checkConfigMap(cm *corev1.ConfigMap, values *Values, namespace string) { - Expect(cm.Name).To(Equal(values.Name)) - Expect(cm.OwnerReferences).To(HaveLen(1)) - Expect(cm.OwnerReferences[0]).To(Equal(values.OwnerReference)) - Expect(cm.Labels).To(Equal(values.Labels)) - configYML := cm.Data[etcdConfig] - config := map[string]interface{}{} - err := yaml.Unmarshal([]byte(configYML), &config) - Expect(err).NotTo(HaveOccurred()) - - Expect(config).To(MatchKeys(IgnoreExtras, Keys{ - "name": Equal(fmt.Sprintf("etcd-%s", values.EtcdUID[:6])), - "data-dir": Equal("/var/etcd/data/new.etcd"), - "metrics": Equal(string(druidv1alpha1.Basic)), - "snapshot-count": Equal(float64(75000)), - "enable-v2": Equal(false), - "quota-backend-bytes": Equal(float64(values.Quota.Value())), - - "client-transport-security": MatchKeys(IgnoreExtras, Keys{ - "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), - "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), - "client-cert-auth": Equal(true), - "trusted-ca-file": Equal("/var/etcd/ssl/client/ca/ca.crt"), - "auto-tls": Equal(false), - }), - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *values.ClientPort)), - "advertise-client-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", values.PeerServiceName, namespace, *values.ClientPort)), - - "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ - "cert-file": Equal("/var/etcd/ssl/peer/server/tls.crt"), - "key-file": Equal("/var/etcd/ssl/peer/server/tls.key"), - "client-cert-auth": Equal(true), - "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), - "auto-tls": Equal(false), - }), - "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *values.ServerPort)), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", values.PeerServiceName, namespace, *values.ServerPort)), - - "initial-cluster-token": Equal("etcd-cluster"), - "initial-cluster-state": Equal("new"), - "auto-compaction-mode": Equal(string(*values.AutoCompactionMode)), - "auto-compaction-retention": Equal(*values.AutoCompactionRetention), - })) - - jsonString, err := json.Marshal(cm.Data) - Expect(err).NotTo(HaveOccurred()) - configMapChecksum := utils.ComputeSHA256Hex(jsonString) - - Expect(configMapChecksum).To(Equal(values.ConfigMapChecksum)) -} diff --git a/pkg/component/etcd/configmap/values.go b/pkg/component/etcd/configmap/values.go deleted file mode 100644 index 219ca5a71..000000000 --- a/pkg/component/etcd/configmap/values.go +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package configmap - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -const ( - defaultClientPort = 2379 - defaultServerPort = 2380 -) - -// Values contains the values to create a configmap -type Values struct { - // Name is the name of the configmap that holds the ETCD config - Name string - // EtcdUID is the UID of the etcd resource. - EtcdUID types.UID - // Metrics defines the level of detail for exported metrics of etcd, specify 'extensive' to include histogram metrics. - Metrics *druidv1alpha1.MetricsLevel - // Quota defines the etcd DB quota. - Quota *resource.Quantity - // InitialCluster is the initial cluster value to bootstrap ETCD. - InitialCluster string - // ClientUrlTLS hold the TLS configuration details for Client Communication - ClientUrlTLS *druidv1alpha1.TLSConfig - // PeerUrlTLS hold the TLS configuration details for Peer Communication - PeerUrlTLS *druidv1alpha1.TLSConfig - //ClientServiceName is name of the etcd client service - ClientServiceName string - // ClientPort holds the client port - ClientPort *int32 - //PeerServiceName is name of the etcd peer service - PeerServiceName string - // ServerPort holds the peer port - ServerPort *int32 - // AutoCompactionMode defines the auto-compaction-mode: 'periodic' or 'revision'. - AutoCompactionMode *druidv1alpha1.CompactionMode - //AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. - AutoCompactionRetention *string - // ConfigMapChecksum is the checksum of deployed configmap - ConfigMapChecksum string - // Labels are the labels of deployed configmap - Labels map[string]string - // OwnerReference is the OwnerReference for the Configmap. - OwnerReference metav1.OwnerReference -} diff --git a/pkg/component/etcd/configmap/values_helper.go b/pkg/component/etcd/configmap/values_helper.go deleted file mode 100644 index 2929e1cef..000000000 --- a/pkg/component/etcd/configmap/values_helper.go +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package configmap - -import ( - "fmt" - "strconv" - "strings" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "k8s.io/utils/pointer" -) - -// GenerateValues generates `configmap.Values` for the configmap component with the given parameters. -func GenerateValues(etcd *druidv1alpha1.Etcd) *Values { - initialCluster := prepareInitialCluster(etcd) - values := &Values{ - Name: etcd.GetConfigMapName(), - EtcdUID: etcd.UID, - Metrics: etcd.Spec.Etcd.Metrics, - Quota: etcd.Spec.Etcd.Quota, - InitialCluster: initialCluster, - ClientUrlTLS: etcd.Spec.Etcd.ClientUrlTLS, - PeerUrlTLS: etcd.Spec.Etcd.PeerUrlTLS, - ClientServiceName: etcd.GetClientServiceName(), - ClientPort: etcd.Spec.Etcd.ClientPort, - PeerServiceName: etcd.GetPeerServiceName(), - ServerPort: etcd.Spec.Etcd.ServerPort, - AutoCompactionMode: etcd.Spec.Common.AutoCompactionMode, - AutoCompactionRetention: etcd.Spec.Common.AutoCompactionRetention, - OwnerReference: etcd.GetAsOwnerReference(), - Labels: etcd.GetDefaultLabels(), - } - return values -} - -func prepareInitialCluster(etcd *druidv1alpha1.Etcd) string { - protocol := "http" - if etcd.Spec.Etcd.PeerUrlTLS != nil { - protocol = "https" - } - - statefulsetReplicas := int(etcd.Spec.Replicas) - - // Form the service name and pod name for mutinode cluster with the help of ETCD name - podName := etcd.GetOrdinalPodName(0) - domaiName := fmt.Sprintf("%s.%s.%s", etcd.GetPeerServiceName(), etcd.Namespace, "svc") - serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))) - - initialCluster := fmt.Sprintf("%s=%s://%s.%s:%s", podName, protocol, podName, domaiName, serverPort) - if statefulsetReplicas > 1 { - // form initial cluster - initialCluster = "" - for i := 0; i < statefulsetReplicas; i++ { - podName = etcd.GetOrdinalPodName(i) - initialCluster = initialCluster + fmt.Sprintf("%s=%s://%s.%s:%s,", podName, protocol, podName, domaiName, serverPort) - } - } - - initialCluster = strings.Trim(initialCluster, ",") - return initialCluster -} diff --git a/pkg/component/etcd/lease/lease.go b/pkg/component/etcd/lease/lease.go deleted file mode 100644 index d9c0f6537..000000000 --- a/pkg/component/etcd/lease/lease.go +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease - -import ( - "context" - - "github.com/go-logr/logr" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Interface provides a facade for operations on leases. -type Interface interface { - gardenercomponent.Deployer -} -type component struct { - client client.Client - namespace string - logger logr.Logger - values Values -} - -func (c *component) Deploy(ctx context.Context) error { - var ( - deltaSnapshotLease = c.emptyLease(c.values.DeltaSnapshotLeaseName) - fullSnapshotLease = c.emptyLease(c.values.FullSnapshotLeaseName) - ) - - if err := c.syncSnapshotLease(ctx, deltaSnapshotLease); err != nil { - return err - } - - if err := c.syncSnapshotLease(ctx, fullSnapshotLease); err != nil { - return err - } - - return c.syncMemberLeases(ctx) -} - -func (c *component) Destroy(ctx context.Context) error { - var ( - deltaSnapshotLease = c.emptyLease(c.values.DeltaSnapshotLeaseName) - fullSnapshotLease = c.emptyLease(c.values.FullSnapshotLeaseName) - ) - - if err := c.deleteSnapshotLease(ctx, deltaSnapshotLease); err != nil { - return err - } - - if err := c.deleteSnapshotLease(ctx, fullSnapshotLease); err != nil { - return err - } - - return c.deleteAllMemberLeases(ctx) -} - -// New creates a new lease deployer instance. -func New(c client.Client, logger logr.Logger, namespace string, values Values) Interface { - return &component{ - client: c, - logger: logger, - namespace: namespace, - values: values, - } -} - -func (c *component) emptyLease(name string) *coordinationv1.Lease { - copyLabels := make(map[string]string, len(c.values.Labels)) - for k, v := range c.values.Labels { - copyLabels[k] = v - } - return &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: c.namespace, - Labels: copyLabels, - }, - } -} diff --git a/pkg/component/etcd/lease/lease_member.go b/pkg/component/etcd/lease/lease_member.go deleted file mode 100644 index a84c5cb94..000000000 --- a/pkg/component/etcd/lease/lease_member.go +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease - -import ( - "context" - "fmt" - - "github.com/gardener/etcd-druid/pkg/utils" - - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/flow" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func (c *component) deleteAllMemberLeases(ctx context.Context) error { - labels := utils.GetMemberLeaseLabels(c.values.EtcdName) - - return c.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(c.namespace), client.MatchingLabels(labels)) -} - -func (c *component) syncMemberLeases(ctx context.Context) error { - var ( - fns []flow.TaskFn - - labels = utils.GetMemberLeaseLabels(c.values.EtcdName) - prefix = c.values.EtcdName - leaseNames = sets.NewString() - ) - - // Patch or create necessary member leases. - for i := 0; i < int(c.values.Replicas); i++ { - leaseName := memberLeaseName(prefix, i) - - lease := c.emptyLease(leaseName) - fns = append(fns, func(ctx context.Context) error { - _, err := controllerutils.GetAndCreateOrMergePatch(ctx, c.client, lease, func() error { - if lease.Labels == nil { - lease.Labels = make(map[string]string) - } - for k, v := range labels { - lease.Labels[k] = v - } - lease.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - return nil - }) - return err - }) - - leaseNames = leaseNames.Insert(leaseName) - } - - leaseList := &coordinationv1.LeaseList{} - if err := c.client.List(ctx, leaseList, client.InNamespace(c.namespace), client.MatchingLabels(labels)); err != nil { - return err - } - - // Clean up superfluous member leases. - for _, lease := range leaseList.Items { - ls := lease - if leaseNames.Has(ls.Name) { - continue - } - fns = append(fns, func(ctx context.Context) error { - if err := c.client.Delete(ctx, &ls); client.IgnoreNotFound(err) != nil { - return err - } - return nil - }) - } - - return flow.Parallel(fns...)(ctx) -} - -func memberLeaseName(etcdName string, replica int) string { - return fmt.Sprintf("%s-%d", etcdName, replica) -} diff --git a/pkg/component/etcd/lease/lease_snapshot.go b/pkg/component/etcd/lease/lease_snapshot.go deleted file mode 100644 index 5eebfcc2a..000000000 --- a/pkg/component/etcd/lease/lease_snapshot.go +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease - -import ( - "context" - - "github.com/gardener/gardener/pkg/controllerutils" - - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func (c *component) deleteSnapshotLease(ctx context.Context, lease *coordinationv1.Lease) error { - return client.IgnoreNotFound(c.client.Delete(ctx, lease)) -} - -func (c *component) syncSnapshotLease(ctx context.Context, lease *coordinationv1.Lease) error { - if !c.values.BackupEnabled { - return c.deleteSnapshotLease(ctx, lease) - } - _, err := controllerutils.GetAndCreateOrMergePatch(ctx, c.client, lease, func() error { - lease.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - return nil - }) - return err -} diff --git a/pkg/component/etcd/lease/lease_suite_test.go b/pkg/component/etcd/lease/lease_suite_test.go deleted file mode 100644 index d8fc05215..000000000 --- a/pkg/component/etcd/lease/lease_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestLease(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Etcd Component Suite") -} diff --git a/pkg/component/etcd/lease/lease_test.go b/pkg/component/etcd/lease/lease_test.go deleted file mode 100644 index 135784460..000000000 --- a/pkg/component/etcd/lease/lease_test.go +++ /dev/null @@ -1,286 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease_test - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - "k8s.io/utils/pointer" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/common" - . "github.com/gardener/etcd-druid/pkg/component/etcd/lease" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - gomegatypes "github.com/onsi/gomega/types" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var _ = Describe("Lease", func() { - var ( - ctx context.Context - c client.Client - - etcd *druidv1alpha1.Etcd - namespace string - name string - uid types.UID - replicas int32 - - values Values - leaseDeployer component.Deployer - ) - - BeforeEach(func() { - ctx = context.Background() - namespace = "default" - name = "etcd-test" - replicas = 3 - uid = "123" - - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: uid, - }, - Spec: druidv1alpha1.EtcdSpec{ - Backup: druidv1alpha1.BackupSpec{ - Store: new(druidv1alpha1.StoreSpec), - }, - Replicas: replicas, - }, - } - - c = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - - values = GenerateValues(etcd) - leaseDeployer = New(c, logr.Discard(), namespace, values) - }) - - Describe("#Deploy", func() { - Context("when deployment is new", func() { - It("should successfully deploy", func() { - Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) - - checkMemberLeases(ctx, c, etcd) - checkSnapshotLeases(ctx, c, etcd, values) - }) - }) - - Context("when deployment is changed from 5 -> 3 replicas", func() { - var etcd2 *druidv1alpha1.Etcd - - BeforeEach(func() { - etcd2 = etcd.DeepCopy() - etcd2.Namespace = "kube-system" - etcd2.Spec.Replicas = 5 - - // Create existing leases - for _, l := range []coordinationv1.Lease{ - memberLease(etcd, 0, false), - memberLease(etcd, 1, false), - memberLease(etcd, 2, false), - memberLease(etcd, 3, false), - memberLease(etcd, 4, false), - // Add leases for another etcd in a different namespace which is not scaled down. - memberLease(etcd2, 0, true), - memberLease(etcd2, 1, true), - memberLease(etcd2, 2, true), - memberLease(etcd2, 3, true), - memberLease(etcd2, 4, true), - } { - lease := l - Expect(c.Create(ctx, &lease)).To(Succeed()) - } - }) - - It("should successfully deploy", func() { - Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) - - checkMemberLeases(ctx, c, etcd) - checkMemberLeases(ctx, c, etcd2) - checkSnapshotLeases(ctx, c, etcd, values) - }) - }) - - Context("when deployment is changed from 1 -> 3 replicas", func() { - BeforeEach(func() { - // Create existing leases - for _, l := range []coordinationv1.Lease{ - memberLease(etcd, 0, false), - } { - lease := l - Expect(c.Create(ctx, &lease)).To(Succeed()) - } - }) - - It("should successfully deploy", func() { - Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) - - checkMemberLeases(ctx, c, etcd) - checkSnapshotLeases(ctx, c, etcd, values) - }) - }) - - Context("when backup is disabled", func() { - BeforeEach(func() { - newValues := GenerateValues(etcd) - newValues.BackupEnabled = false // backup is disabled - leaseDeployer = New(c, logr.Discard(), namespace, newValues) - }) - - It("should successfully deploy", func() { - // Snapshot leases might exist before, so create them here. - for _, l := range []coordinationv1.Lease{ - {ObjectMeta: metav1.ObjectMeta{Name: values.DeltaSnapshotLeaseName}}, - {ObjectMeta: metav1.ObjectMeta{Name: values.FullSnapshotLeaseName}}, - } { - lease := l - Expect(c.Create(ctx, &lease)).To(Succeed()) - } - - Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) - - checkMemberLeases(ctx, c, etcd) - - _, err := getSnapshotLease(ctx, c, namespace, values.DeltaSnapshotLeaseName) - Expect(err).To(matchers.BeNotFoundError()) - _, err = getSnapshotLease(ctx, c, namespace, values.FullSnapshotLeaseName) - Expect(err).To(matchers.BeNotFoundError()) - }) - }) - }) - - Describe("#Destroy", func() { - Context("when no leases exist", func() { - It("should destroy without errors", func() { - Expect(leaseDeployer.Destroy(ctx)).To(Succeed()) - }) - }) - - Context("when leases exist", func() { - It("should destroy without errors", func() { - for _, l := range []coordinationv1.Lease{ - memberLease(etcd, 0, false), - memberLease(etcd, 1, false), - memberLease(etcd, 2, false), - {ObjectMeta: metav1.ObjectMeta{Name: values.DeltaSnapshotLeaseName}}, - {ObjectMeta: metav1.ObjectMeta{Name: values.FullSnapshotLeaseName}}, - } { - lease := l - Expect(c.Create(ctx, &lease)).To(Succeed()) - } - - Expect(leaseDeployer.Destroy(ctx)).To(Succeed()) - - _, err := getSnapshotLease(ctx, c, namespace, values.DeltaSnapshotLeaseName) - Expect(err).To(matchers.BeNotFoundError()) - _, err = getSnapshotLease(ctx, c, namespace, values.FullSnapshotLeaseName) - Expect(err).To(matchers.BeNotFoundError()) - - leases := &coordinationv1.LeaseList{} - Expect(c.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels(map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-member-lease", - }))).To(Succeed()) - - Expect(leases.Items).To(BeEmpty()) - }) - }) - }) -}) - -func getSnapshotLease(ctx context.Context, c client.Client, namespace, name string) (*coordinationv1.Lease, error) { - snapshotLease := coordinationv1.Lease{} - if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &snapshotLease); err != nil { - return nil, err - } - return &snapshotLease, nil -} - -func checkSnapshotLeases(ctx context.Context, c client.Client, etcd *druidv1alpha1.Etcd, val Values) { - deltaLease, err := getSnapshotLease(ctx, c, etcd.Namespace, val.DeltaSnapshotLeaseName) - Expect(err).NotTo(HaveOccurred()) - Expect(deltaLease).To(PointTo(matchLeaseElement(deltaLease.Name, etcd.Name, etcd.UID))) - - fullLease, err := getSnapshotLease(ctx, c, etcd.Namespace, val.FullSnapshotLeaseName) - Expect(err).NotTo(HaveOccurred()) - Expect(fullLease).To(PointTo(matchLeaseElement(fullLease.Name, etcd.Name, etcd.UID))) -} - -func checkMemberLeases(ctx context.Context, c client.Client, etcd *druidv1alpha1.Etcd) { - leases := &coordinationv1.LeaseList{} - Expect(c.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels(map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-member-lease", - }))).To(Succeed()) - Expect(leases.Items).To(ConsistOf(memberLeases(etcd.Name, etcd.UID, etcd.Spec.Replicas))) -} - -func memberLeases(name string, etcdUID types.UID, replicas int32) []interface{} { - var elements []interface{} - for i := 0; i < int(replicas); i++ { - elements = append(elements, matchLeaseElement(fmt.Sprintf("%s-%d", name, i), name, etcdUID)) - } - - return elements -} - -func matchLeaseElement(leaseName, etcdName string, etcdUID types.UID) gomegatypes.GomegaMatcher { - return MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(leaseName), - "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(etcdName), - "UID": Equal(etcdUID), - "Controller": PointTo(BeTrue()), - "BlockOwnerDeletion": PointTo(BeTrue()), - })), - }), - }) -} - -func memberLease(etcd *druidv1alpha1.Etcd, replica int, withOwnerRef bool) coordinationv1.Lease { - lease := coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", etcd.Name, replica), - Namespace: etcd.Namespace, - Labels: map[string]string{ - common.GardenerOwnedBy: etcd.Name, - v1beta1constants.GardenerPurpose: "etcd-member-lease", - }, - }, - } - - if withOwnerRef { - lease.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ - { - APIVersion: druidv1alpha1.GroupVersion.String(), - Kind: "Etcd", - Name: etcd.Name, - UID: etcd.UID, - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - }, - } - } - - return lease -} diff --git a/pkg/component/etcd/lease/values.go b/pkg/component/etcd/lease/values.go deleted file mode 100644 index 54362cdd1..000000000 --- a/pkg/component/etcd/lease/values.go +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease - -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - -// Values contains the values necessary for creating ETCD leases. -type Values struct { - // BackupEnabled specifies if the backup functionality for the etcd cluster is enabled. - BackupEnabled bool - // EtcdName is the name of the etcd resource. - EtcdName string - // DeltaSnapshotLeaseName is the name of the delta snapshot lease object. - DeltaSnapshotLeaseName string - // FullSnapshotLeaseName is the name of the full snapshot lease object. - FullSnapshotLeaseName string - // Replicas is the replica count of the etcd cluster. - Replicas int32 - // Labels is the labels of deployed configmap - Labels map[string]string - // OwnerReference is the OwnerReference for the Configmap. - OwnerReference metav1.OwnerReference -} diff --git a/pkg/component/etcd/lease/values_helper.go b/pkg/component/etcd/lease/values_helper.go deleted file mode 100644 index 921f1688f..000000000 --- a/pkg/component/etcd/lease/values_helper.go +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package lease - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -) - -// GenerateValues generates `lease.Values` for the lease component -func GenerateValues(etcd *druidv1alpha1.Etcd) Values { - return Values{ - BackupEnabled: etcd.Spec.Backup.Store != nil, - EtcdName: etcd.Name, - DeltaSnapshotLeaseName: etcd.GetDeltaSnapshotLeaseName(), - FullSnapshotLeaseName: etcd.GetFullSnapshotLeaseName(), - Replicas: etcd.Spec.Replicas, - Labels: etcd.GetDefaultLabels(), - OwnerReference: etcd.GetAsOwnerReference(), - } -} diff --git a/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget.go b/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget.go deleted file mode 100644 index 8b326b8c6..000000000 --- a/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget.go +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package poddisruptionbudget - -import ( - "context" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/controllerutils" - policyv1 "k8s.io/api/policy/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type component struct { - client client.Client - namespace string - values *Values -} - -// New creates a new poddisruptionbudget deployer instance. -func New(c client.Client, namespace string, values *Values) gardenercomponent.Deployer { - return &component{ - client: c, - namespace: namespace, - values: values, - } -} - -// emptyPodDisruptionBudget returns an empty PDB object with only the name and namespace as part of the object meta -func (c *component) emptyPodDisruptionBudget() *policyv1.PodDisruptionBudget { - return &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.Name, - Namespace: c.namespace, - }, - } -} - -// Deploy creates a PDB or synchronizes the PDB spec based on the etcd spec -func (c *component) Deploy(ctx context.Context) error { - pdb := c.emptyPodDisruptionBudget() - - return c.syncPodDisruptionBudget(ctx, pdb) -} - -// syncPodDisruptionBudget Creates a PDB if it does not exist -// Patches the PDB spec if a PDB already exist -func (c *component) syncPodDisruptionBudget(ctx context.Context, pdb *policyv1.PodDisruptionBudget) error { - _, err := controllerutils.GetAndCreateOrMergePatch(ctx, c.client, pdb, func() error { - pdb.Labels = c.values.Labels - pdb.Annotations = c.values.Annotations - pdb.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - pdb.Spec.MinAvailable = &intstr.IntOrString{ - IntVal: c.values.MinAvailable, - Type: intstr.Int, - } - pdb.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: c.values.SelectorLabels, - } - return nil - }) - return err -} - -// Destroy deletes a PDB. Ignores if PDB does not exist -func (c *component) Destroy(ctx context.Context) error { - pdb := c.emptyPodDisruptionBudget() - - return client.IgnoreNotFound(c.client.Delete(ctx, pdb)) -} diff --git a/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_suite_test.go b/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_suite_test.go deleted file mode 100644 index 82188b520..000000000 --- a/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package poddisruptionbudget - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestPodDisruptionBudget(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "PodDisruptionBudget Component Suite") -} diff --git a/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_test.go b/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_test.go deleted file mode 100644 index f5184ebee..000000000 --- a/pkg/component/etcd/poddisruptionbudget/poddisruptionbudget_test.go +++ /dev/null @@ -1,226 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package poddisruptionbudget_test - -import ( - "context" - - . "github.com/gardener/etcd-druid/pkg/component/etcd/poddisruptionbudget" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/gardener/gardener/pkg/component" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func GetFakeKubernetesClientSet() client.Client { - return fake.NewClientBuilder().Build() -} - -var _ = Describe("PodDisruptionBudget", func() { - var ( - ctx context.Context - cl client.Client - - defaultPDB *policyv1.PodDisruptionBudget - etcd *druidv1alpha1.Etcd - namespace string - name string - uid types.UID - - metricsLevel druidv1alpha1.MetricsLevel - quota resource.Quantity - labels map[string]string - - values Values - pdbDeployer component.Deployer - ) - - BeforeEach(func() { - ctx = context.Background() - cl = GetFakeKubernetesClientSet() - - name = "poddisruptionbudget" - namespace = "default" - uid = "12345678" - labels = map[string]string{ - "role": "main", - } - - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: uid, - }, - Spec: druidv1alpha1.EtcdSpec{ - Labels: labels, - Selector: metav1.SetAsLabelSelector(labels), - Replicas: 3, - Etcd: druidv1alpha1.EtcdConfig{ - Quota: "a, - Metrics: &metricsLevel, - ClientUrlTLS: &druidv1alpha1.TLSConfig{ - ClientTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-client-tls", - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-server-cert", - }, - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "client-url-ca-etcd", - }, - }, - }, - PeerUrlTLS: &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "peer-url-ca-etcd", - }, - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "peer-url-etcd-server-tls", - }, - }, - }, - }, - } - - defaultPDB = &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - MinAvailable: &intstr.IntOrString{ - Type: 0, - IntVal: 0, - }, - }, - } - }) - Describe("#Deploy", func() { - var ( - pdb *policyv1.PodDisruptionBudget - ) - BeforeEach(func() { - pdb = &policyv1.PodDisruptionBudget{} - }) - AfterEach(func() { - Expect(cl.Delete(ctx, pdb)).To(Succeed()) - }) - Context("when PDB does not exist", func() { - Context("when etcd replicas are 0", func() { - It("should create the PDB successfully", func() { - etcd.Spec.Replicas = 0 - etcd.Status = druidv1alpha1.EtcdStatus{} - values = GenerateValues(etcd) - pdbDeployer = New(cl, namespace, &values) - Expect(pdbDeployer.Deploy(ctx)).To(Succeed()) - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), pdb)).To(Succeed()) - Expect(pdb.Spec.MinAvailable.IntVal).To(BeNumerically("==", 0)) - checkPDB(pdb, &values, namespace) - }) - }) - Context("when etcd replicas are 5", func() { - It("should create the PDB successfully", func() { - etcd.Spec.Replicas = 5 - values = GenerateValues(etcd) - pdbDeployer = New(cl, namespace, &values) - Expect(pdbDeployer.Deploy(ctx)).To(Succeed()) - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), pdb)).To(Succeed()) - Expect(pdb.Spec.MinAvailable.IntVal).To(BeNumerically("==", 3)) - checkPDB(pdb, &values, namespace) - }) - }) - }) - - Context("when pdb already exists", func() { - It("should update the pdb successfully when etcd replicas change", func() { - // existing PDB with min = 0 - values = GenerateValues(etcd) - Expect(cl.Create(ctx, defaultPDB)).To(Succeed()) - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), pdb)).To(Succeed()) - Expect(pdb.Spec.MinAvailable.IntVal).To(BeNumerically("==", 0)) - - // Post updating the etcd replicas - etcd.Spec.Replicas = 5 - values = GenerateValues(etcd) - pdbDeployer = New(cl, namespace, &values) - Expect(pdbDeployer.Deploy(ctx)).To(Succeed()) - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), pdb)).To(Succeed()) - Expect(pdb.Spec.MinAvailable.IntVal).To(BeNumerically("==", 3)) - checkPDB(pdb, &values, namespace) - }) - }) - }) - - Describe("#Destroy", func() { - var ( - pdb *policyv1.PodDisruptionBudget - values Values - ) - BeforeEach(func() { - pdb = &policyv1.PodDisruptionBudget{} - values = GenerateValues(etcd) - }) - Context("when PDB does not exist", func() { - It("should destroy successfully ignoring the not-found error", func() { - pdbDeployer = New(cl, namespace, &values) - Expect(pdbDeployer.Destroy(ctx)).To(Succeed()) - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), pdb)).To(BeNotFoundError()) - }) - }) - - Context("when PDB exists", func() { - It("should destroy successfully", func() { - Expect(cl.Create(ctx, defaultPDB)).To(Succeed()) - pdbDeployer = New(cl, namespace, &values) - Expect(pdbDeployer.Destroy(ctx)).To(Succeed()) - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), pdb)).To(BeNotFoundError()) - }) - }) - }) -}) - -func checkPDB(pdb *policyv1.PodDisruptionBudget, values *Values, expectedNamespace string) { - Expect(pdb.Name).To(Equal(values.Name)) - Expect(pdb.Namespace).To(Equal(expectedNamespace)) - Expect(pdb.OwnerReferences).To(Equal([]metav1.OwnerReference{values.OwnerReference})) - Expect(pdb.Labels).To(Equal(pdbLabels(values))) - Expect(pdb.Spec.MinAvailable.IntVal).To(BeNumerically("==", values.MinAvailable)) - Expect(pdb.Spec.Selector).To(Equal(pdbSelectorLabels(values))) -} - -func pdbLabels(val *Values) map[string]string { - return map[string]string{ - "name": "etcd", - "instance": val.Name, - } -} - -func pdbSelectorLabels(val *Values) *metav1.LabelSelector { - return &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "instance": val.Name, - "name": "etcd", - }, - } -} diff --git a/pkg/component/etcd/poddisruptionbudget/values.go b/pkg/component/etcd/poddisruptionbudget/values.go deleted file mode 100644 index 7b47ed700..000000000 --- a/pkg/component/etcd/poddisruptionbudget/values.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package poddisruptionbudget - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - instanceKey = "instance" - nameKey = "name" - appKey = "app" -) - -// Values contains the values necessary for creating PodDisruptionBudget. -type Values struct { - // Name is the name of the PodDisruptionBudget. - Name string - // Labels are the PDB labels. - Labels map[string]string - // SelectorLabels are the labels to be used in the PDB spec selector - SelectorLabels map[string]string - // Annotations are the annotations to be used in the PDB - Annotations map[string]string - // MinAvailable defined the minimum number of pods to be available at any point of time - MinAvailable int32 - // OwnerReference is the OwnerReference for the PodDisruptionBudget. - OwnerReference metav1.OwnerReference -} diff --git a/pkg/component/etcd/poddisruptionbudget/values_helper.go b/pkg/component/etcd/poddisruptionbudget/values_helper.go deleted file mode 100644 index 8d8c44ab0..000000000 --- a/pkg/component/etcd/poddisruptionbudget/values_helper.go +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package poddisruptionbudget - -import ( - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" -) - -// GenerateValues generates `poddisruptionbudget.Values` for the lease component with the given parameters. -func GenerateValues(etcd *druidv1alpha1.Etcd) Values { - annotations := map[string]string{ - common.GardenerOwnedBy: fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - common.GardenerOwnerType: "etcd", - } - - return Values{ - Name: etcd.Name, - Labels: etcd.GetDefaultLabels(), - SelectorLabels: etcd.GetDefaultLabels(), - Annotations: annotations, - MinAvailable: int32(CalculatePDBMinAvailable(etcd)), - OwnerReference: etcd.GetAsOwnerReference(), - } -} - -// CalculatePDBMinAvailable calculates the minimum available value for the PDB -func CalculatePDBMinAvailable(etcd *druidv1alpha1.Etcd) int { - // do not enable for single node cluster - if etcd.Spec.Replicas <= 1 { - return 0 - } - - clusterSize := int(etcd.Spec.Replicas) - return clusterSize/2 + 1 - -} diff --git a/pkg/component/etcd/poddisruptionbudget/values_helper_test.go b/pkg/component/etcd/poddisruptionbudget/values_helper_test.go deleted file mode 100644 index 39b59ae6b..000000000 --- a/pkg/component/etcd/poddisruptionbudget/values_helper_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package poddisruptionbudget_test - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/component/etcd/poddisruptionbudget" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var _ = Describe("PodDisruptionBudget", func() { - var ( - etcd *druidv1alpha1.Etcd - ) - - BeforeEach(func() { - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd", - Namespace: "default", - }, - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 3, - }, - } - }) - Describe("#CalculatePDBMinAvailable", func() { - Context("With etcd replicas less than 2", func() { - It("Should return MinAvailable 0 if replicas is set to 1", func() { - etcd.Spec.Replicas = 1 - Expect(CalculatePDBMinAvailable(etcd)).To(Equal(0)) - }) - }) - Context("With etcd replicas more than 2", func() { - It("Should return MinAvailable equal to quorum size of cluster", func() { - etcd.Spec.Replicas = 5 - Expect(CalculatePDBMinAvailable(etcd)).To(Equal(3)) - }) - It("Should return MinAvailable equal to quorum size of cluster", func() { - etcd.Spec.Replicas = 6 - Expect(CalculatePDBMinAvailable(etcd)).To(Equal(4)) - }) - }) - }) -}) diff --git a/pkg/component/etcd/role/role.go b/pkg/component/etcd/role/role.go deleted file mode 100644 index a8e2b6fe2..000000000 --- a/pkg/component/etcd/role/role.go +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package role - -import ( - "context" - - "github.com/gardener/gardener/pkg/controllerutils" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type component struct { - client client.Client - values *Values -} - -func (c component) Deploy(ctx context.Context) error { - role := c.emptyRole() - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, role, func() error { - role.Name = c.values.Name - role.Namespace = c.values.Namespace - role.Labels = c.values.Labels - role.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - role.Rules = c.values.Rules - return nil - }) - return err -} - -func (c component) Destroy(ctx context.Context) error { - return client.IgnoreNotFound(c.client.Delete(ctx, c.emptyRole())) -} - -func (c component) emptyRole() *rbacv1.Role { - return &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.Name, - Namespace: c.values.Namespace, - }, - } -} - -// New creates a new role deployer instance. -func New(c client.Client, values *Values) gardenercomponent.Deployer { - return &component{ - client: c, - values: values, - } -} diff --git a/pkg/component/etcd/role/role_suite_test.go b/pkg/component/etcd/role/role_suite_test.go deleted file mode 100644 index 4bf325453..000000000 --- a/pkg/component/etcd/role/role_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package role - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestService(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Role Component Suite") -} diff --git a/pkg/component/etcd/role/role_test.go b/pkg/component/etcd/role/role_test.go deleted file mode 100644 index 762b23bab..000000000 --- a/pkg/component/etcd/role/role_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package role_test - -import ( - "context" - - "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/component/etcd/role" - - "github.com/gardener/gardener/pkg/component" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" - - testutils "github.com/gardener/etcd-druid/test/utils" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var _ = Describe("Role Component", Ordered, func() { - var ( - ctx context.Context - c client.Client - values *role.Values - roleComponent component.Deployer - ) - - BeforeEach(func() { - ctx = context.Background() - c = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - values = getTestRoleValues() - roleComponent = role.New(c, values) - }) - - Describe("#Deploy", func() { - It("should create the Role with the expected values", func() { - By("creating a Role") - err := roleComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that the Role is created on the K8s cluster as expected") - created := &rbacv1.Role{} - err = c.Get(ctx, getRoleKeyFromValue(values), created) - Expect(err).NotTo(HaveOccurred()) - verifyRoleValues(created, values) - }) - It("should update the Role with the expected values", func() { - By("updating the Role") - values.Labels["new"] = "label" - err := roleComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that the Role is updated on the K8s cluster as expected") - updated := &rbacv1.Role{} - err = c.Get(ctx, getRoleKeyFromValue(values), updated) - Expect(err).NotTo(HaveOccurred()) - verifyRoleValues(updated, values) - }) - It("should not return an error when there is nothing to update the Role", func() { - err := roleComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - updated := &rbacv1.Role{} - err = c.Get(ctx, getRoleKeyFromValue(values), updated) - Expect(err).NotTo(HaveOccurred()) - verifyRoleValues(updated, values) - }) - It("should return an error when the update fails", func() { - values.Name = "" - err := roleComponent.Deploy(ctx) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Required value: name is required")) - }) - }) - - Describe("#Destroy", func() { - It("should delete the Role", func() { - By("deleting the Role") - err := roleComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that the Role is deleted from the K8s cluster as expected") - role := &rbacv1.Role{} - Expect(c.Get(ctx, getRoleKeyFromValue(values), role)).To(BeNotFoundError()) - }) - It("should not return an error when there is nothing to delete", func() { - err := roleComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - }) -}) - -func verifyRoleValues(expected *rbacv1.Role, values *role.Values) { - Expect(expected.Name).To(Equal(values.Name)) - Expect(expected.Labels).To(Equal(values.Labels)) - Expect(expected.Namespace).To(Equal(values.Namespace)) - Expect(expected.OwnerReferences).To(Equal([]metav1.OwnerReference{values.OwnerReference})) - Expect(expected.Rules).To(MatchAllElements(testutils.RuleIterator, Elements{ - "coordination.k8s.io": MatchFields(IgnoreExtras, Fields{ - "APIGroups": MatchAllElements(testutils.StringArrayIterator, Elements{ - "coordination.k8s.io": Equal("coordination.k8s.io"), - }), - "Resources": MatchAllElements(testutils.StringArrayIterator, Elements{ - "leases": Equal("leases"), - }), - "Verbs": MatchAllElements(testutils.StringArrayIterator, Elements{ - "list": Equal("list"), - "get": Equal("get"), - "update": Equal("update"), - "patch": Equal("patch"), - "watch": Equal("watch"), - }), - }), - "apps": MatchFields(IgnoreExtras, Fields{ - "APIGroups": MatchAllElements(testutils.StringArrayIterator, Elements{ - "apps": Equal("apps"), - }), - "Resources": MatchAllElements(testutils.StringArrayIterator, Elements{ - "statefulsets": Equal("statefulsets"), - }), - "Verbs": MatchAllElements(testutils.StringArrayIterator, Elements{ - "list": Equal("list"), - "get": Equal("get"), - "update": Equal("update"), - "patch": Equal("patch"), - "watch": Equal("watch"), - }), - }), - "": MatchFields(IgnoreExtras, Fields{ - "APIGroups": MatchAllElements(testutils.StringArrayIterator, Elements{ - "": Equal(""), - }), - "Resources": MatchAllElements(testutils.StringArrayIterator, Elements{ - "pods": Equal("pods"), - }), - "Verbs": MatchAllElements(testutils.StringArrayIterator, Elements{ - "list": Equal("list"), - "get": Equal("get"), - "watch": Equal("watch"), - }), - }), - })) -} -func getRoleKeyFromValue(values *role.Values) types.NamespacedName { - return client.ObjectKey{Name: values.Name, Namespace: values.Namespace} -} - -func getTestRoleValues() *role.Values { - return &role.Values{ - Name: "test-role", - Namespace: "test-namespace", - Labels: map[string]string{ - "foo": "bar", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, - OwnerReference: metav1.OwnerReference{ - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "etcd", - Name: "test-etcd", - UID: "123-456-789", - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - }, - } -} diff --git a/pkg/component/etcd/role/values.go b/pkg/component/etcd/role/values.go deleted file mode 100644 index be39ea2a4..000000000 --- a/pkg/component/etcd/role/values.go +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package role - -import ( - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Values defines the fields used to create a Role for Etcd. -type Values struct { - // Name is the name of the Role. - Name string - // Namespace is the namespace of the Role. - Namespace string - // Rules holds all the PolicyRules for this Role - Rules []rbacv1.PolicyRule - // OwnerReference is the OwnerReference of the Role. - OwnerReference metav1.OwnerReference - // Labels are the labels of the Role. - Labels map[string]string -} diff --git a/pkg/component/etcd/role/values_helper.go b/pkg/component/etcd/role/values_helper.go deleted file mode 100644 index 54f291c95..000000000 --- a/pkg/component/etcd/role/values_helper.go +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package role - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - rbacv1 "k8s.io/api/rbac/v1" -) - -// GenerateValues generates `role.Values` for the role component with the given `etcd` object. -func GenerateValues(etcd *druidv1alpha1.Etcd) *Values { - return &Values{ - Name: etcd.GetRoleName(), - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReference: etcd.GetAsOwnerReference(), - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, - } -} diff --git a/pkg/component/etcd/role/values_helper_test.go b/pkg/component/etcd/role/values_helper_test.go deleted file mode 100644 index d0b48fbd8..000000000 --- a/pkg/component/etcd/role/values_helper_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package role_test - -import ( - "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/component/etcd/role" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var _ = Describe("Role", func() { - var ( - etcd = &v1alpha1.Etcd{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "Etcd", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd-name", - Namespace: "etcd-namespace", - UID: "etcd-uid", - }, - Spec: v1alpha1.EtcdSpec{ - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - } - expected = &role.Values{ - Name: etcd.GetRoleName(), - Namespace: etcd.Namespace, - Labels: map[string]string{ - "name": "etcd", - "instance": etcd.Name, - }, - OwnerReference: etcd.GetAsOwnerReference(), - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, - } - ) - - Context("Generate Values", func() { - It("should generate correct values", func() { - values := role.GenerateValues(etcd) - Expect(values).To(Equal(expected)) - }) - }) -}) diff --git a/pkg/component/etcd/rolebinding/rolebinding.go b/pkg/component/etcd/rolebinding/rolebinding.go deleted file mode 100644 index 5c026af69..000000000 --- a/pkg/component/etcd/rolebinding/rolebinding.go +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package rolebinding - -import ( - "context" - - "github.com/gardener/gardener/pkg/controllerutils" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type component struct { - client client.Client - values *Values -} - -func (c component) Deploy(ctx context.Context) error { - roleBinding := c.emptyRoleBinding() - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, roleBinding, func() error { - roleBinding.Name = c.values.Name - roleBinding.Namespace = c.values.Namespace - roleBinding.Labels = c.values.Labels - roleBinding.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - roleBinding.RoleRef = rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: c.values.RoleName, - } - roleBinding.Subjects = []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: c.values.ServiceAccountName, - Namespace: c.values.Namespace, - }, - } - return nil - }) - return err -} - -func (c component) Destroy(ctx context.Context) error { - return client.IgnoreNotFound(c.client.Delete(ctx, c.emptyRoleBinding())) -} - -func (c *component) emptyRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.Name, - Namespace: c.values.Namespace, - }, - } -} - -// New creates a new role binding deployer instance. -func New(c client.Client, value *Values) gardenercomponent.Deployer { - return &component{ - client: c, - values: value, - } -} diff --git a/pkg/component/etcd/rolebinding/rolebinding_suite_test.go b/pkg/component/etcd/rolebinding/rolebinding_suite_test.go deleted file mode 100644 index 95b1703c7..000000000 --- a/pkg/component/etcd/rolebinding/rolebinding_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package rolebinding - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestRoleBinding(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "RoleBinding Component Suite") -} diff --git a/pkg/component/etcd/rolebinding/rolebinding_test.go b/pkg/component/etcd/rolebinding/rolebinding_test.go deleted file mode 100644 index 6b33b6ff8..000000000 --- a/pkg/component/etcd/rolebinding/rolebinding_test.go +++ /dev/null @@ -1,165 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package rolebinding_test - -import ( - "context" - - "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/component/etcd/rolebinding" - - "github.com/gardener/gardener/pkg/component" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var _ = Describe("RoleBinding Component", func() { - var ( - ctx context.Context - c client.Client - values *rolebinding.Values - roleBindingComponent component.Deployer - ) - - Context("#Deploy", func() { - - BeforeEach(func() { - ctx = context.Background() - c = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - values = getTestRoleBindingValues() - roleBindingComponent = rolebinding.New(c, values) - - err := roleBindingComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - err := roleBindingComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - - It("should create the RoleBinding with the expected values", func() { - By("verifying that the RoleBinding is created on the K8s cluster as expected") - created := &rbacv1.RoleBinding{} - err := c.Get(ctx, getRoleBindingKeyFromValue(values), created) - Expect(err).NotTo(HaveOccurred()) - verifyRoleBindingValues(created, values) - }) - - It("should update the RoleBinding with the expected values", func() { - By("updating the RoleBinding") - values.Labels["new"] = "label" - err := roleBindingComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that the RoleBinding is updated on the K8s cluster as expected") - updated := &rbacv1.RoleBinding{} - err = c.Get(ctx, getRoleBindingKeyFromValue(values), updated) - Expect(err).NotTo(HaveOccurred()) - verifyRoleBindingValues(updated, values) - }) - - It("should not return an error when there is nothing to update the RoleBinding", func() { - err := roleBindingComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - updated := &rbacv1.RoleBinding{} - err = c.Get(ctx, getRoleBindingKeyFromValue(values), updated) - Expect(err).NotTo(HaveOccurred()) - verifyRoleBindingValues(updated, values) - }) - - It("should return an error when the update fails", func() { - values.Name = "" - err := roleBindingComponent.Deploy(ctx) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Required value: name is required")) - }) - }) - - Context("#Destroy", func() { - - BeforeEach(func() { - ctx = context.Background() - c = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - values = getTestRoleBindingValues() - - roleBindingComponent = rolebinding.New(c, values) - err := roleBindingComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - - It("should delete the RoleBinding", func() { - By("deleting the RoleBinding") - err := roleBindingComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - - roleBinding := &rbacv1.RoleBinding{} - By("verifying that the RoleBinding is deleted from the K8s cluster as expected") - Expect(c.Get(ctx, getRoleBindingKeyFromValue(values), roleBinding)).To(BeNotFoundError()) - }) - - It("should not return an error when there is nothing to delete", func() { - By("deleting the RoleBinding") - err := roleBindingComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that attempting to delete the RoleBinding again returns no error") - err = roleBindingComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - }) -}) - -func verifyRoleBindingValues(expected *rbacv1.RoleBinding, values *rolebinding.Values) { - Expect(expected.Name).To(Equal(values.Name)) - Expect(expected.Labels).To(Equal(values.Labels)) - Expect(expected.Namespace).To(Equal(values.Namespace)) - Expect(expected.Namespace).To(Equal(values.Namespace)) - Expect(expected.OwnerReferences).To(Equal([]metav1.OwnerReference{values.OwnerReference})) - Expect(expected.Subjects).To(Equal([]rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: values.ServiceAccountName, - Namespace: values.Namespace, - }, - })) - Expect(expected.RoleRef).To(Equal(rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: values.RoleName, - })) -} -func getRoleBindingKeyFromValue(values *rolebinding.Values) types.NamespacedName { - return client.ObjectKey{Name: values.Name, Namespace: values.Namespace} -} - -func getTestRoleBindingValues() *rolebinding.Values { - return &rolebinding.Values{ - Name: "test-rolebinding", - Namespace: "test-namespace", - Labels: map[string]string{ - "foo": "bar", - }, - RoleName: "test-role", - ServiceAccountName: "test-serviceaccount", - OwnerReference: metav1.OwnerReference{ - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "etcd", - Name: "test-etcd", - UID: "123-456-789", - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - }, - } -} diff --git a/pkg/component/etcd/rolebinding/values.go b/pkg/component/etcd/rolebinding/values.go deleted file mode 100644 index bc1f8290d..000000000 --- a/pkg/component/etcd/rolebinding/values.go +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package rolebinding - -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - -// Values defines the fields used to create a RoleBinding for Etcd. -type Values struct { - // Name is the name of the RoleBinding. - Name string - // Namespace is the namespace of the RoleBinding. - Namespace string - // RoleName is the role name of the RoleBinding. It is assumed that the role exists in the namespace where the etcd custom resource is created. - RoleName string - // ServiceAccountName is the service account subject name for the RoleBinding. - // It is assumed that the ServiceAccount exists in the namespace where the etcd custom resource is created. - ServiceAccountName string - // OwnerReference is the OwnerReference for the RoleBinding. - OwnerReference metav1.OwnerReference - // Labels are the labels of the RoleBinding. - Labels map[string]string -} diff --git a/pkg/component/etcd/rolebinding/values_helper.go b/pkg/component/etcd/rolebinding/values_helper.go deleted file mode 100644 index c0c1d6c79..000000000 --- a/pkg/component/etcd/rolebinding/values_helper.go +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package rolebinding - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -) - -// GenerateValues generates `serviceaccount.Values` for the serviceaccount component with the given `etcd` object. -func GenerateValues(etcd *druidv1alpha1.Etcd) *Values { - return &Values{ - Name: etcd.GetRoleBindingName(), - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReference: etcd.GetAsOwnerReference(), - RoleName: etcd.GetRoleName(), - ServiceAccountName: etcd.GetServiceAccountName(), - } -} diff --git a/pkg/component/etcd/rolebinding/values_helper_test.go b/pkg/component/etcd/rolebinding/values_helper_test.go deleted file mode 100644 index ec936ed51..000000000 --- a/pkg/component/etcd/rolebinding/values_helper_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package rolebinding_test - -import ( - "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/component/etcd/rolebinding" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var _ = Describe("RoleBinding", func() { - var ( - etcd = &v1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd-name", - Namespace: "etcd-namespace", - UID: "etcd-uid", - }, - Spec: v1alpha1.EtcdSpec{ - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - } - expected = &rolebinding.Values{ - Name: etcd.GetRoleBindingName(), - Namespace: etcd.Namespace, - Labels: map[string]string{ - "name": "etcd", - "instance": etcd.Name, - }, - RoleName: etcd.GetRoleName(), - ServiceAccountName: etcd.GetServiceAccountName(), - OwnerReference: etcd.GetAsOwnerReference(), - } - ) - - Context("Generate Values", func() { - It("should generate correct values", func() { - values := rolebinding.GenerateValues(etcd) - Expect(values).To(Equal(expected)) - }) - }) -}) diff --git a/pkg/component/etcd/service/service.go b/pkg/component/etcd/service/service.go deleted file mode 100644 index 47fb8695c..000000000 --- a/pkg/component/etcd/service/service.go +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type component struct { - client client.Client - namespace string - - values Values -} - -func (c *component) Deploy(ctx context.Context) error { - var ( - clientService = c.emptyService(c.values.ClientServiceName) - peerService = c.emptyService(c.values.PeerServiceName) - ) - - if err := c.syncClientService(ctx, clientService); err != nil { - return err - } - - return c.syncPeerService(ctx, peerService) -} - -func (c *component) Destroy(ctx context.Context) error { - var ( - clientService = c.emptyService(c.values.ClientServiceName) - peerService = c.emptyService(c.values.PeerServiceName) - ) - - if err := client.IgnoreNotFound(c.client.Delete(ctx, clientService)); err != nil { - return err - } - - return client.IgnoreNotFound(c.client.Delete(ctx, peerService)) -} - -// New creates a new service deployer instance. -func New(c client.Client, namespace string, values Values) gardenercomponent.Deployer { - return &component{ - client: c, - namespace: namespace, - values: values, - } -} - -func (c *component) emptyService(name string) *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: c.namespace, - }, - } -} diff --git a/pkg/component/etcd/service/service_client.go b/pkg/component/etcd/service/service_client.go deleted file mode 100644 index 2802ae114..000000000 --- a/pkg/component/etcd/service/service_client.go +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - - "github.com/gardener/etcd-druid/pkg/utils" - "github.com/gardener/gardener/pkg/controllerutils" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func (c *component) syncClientService(ctx context.Context, svc *corev1.Service) error { - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, svc, func() error { - svc.Labels = utils.MergeStringMaps(c.values.Labels, c.values.ClientServiceLabels) - svc.Annotations = c.values.ClientServiceAnnotations - svc.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - svc.Spec.Type = corev1.ServiceTypeClusterIP - svc.Spec.SessionAffinity = corev1.ServiceAffinityNone - svc.Spec.Selector = c.values.SelectorLabels - svc.Spec.Ports = []corev1.ServicePort{ - { - Name: "client", - Protocol: corev1.ProtocolTCP, - Port: c.values.ClientPort, - TargetPort: intstr.FromInt(int(c.values.ClientPort)), - }, - // TODO: Remove the "server" port in a future release - { - Name: "server", - Protocol: corev1.ProtocolTCP, - Port: c.values.PeerPort, - TargetPort: intstr.FromInt(int(c.values.PeerPort)), - }, - { - Name: "backuprestore", - Protocol: corev1.ProtocolTCP, - Port: c.values.BackupPort, - TargetPort: intstr.FromInt(int(c.values.BackupPort)), - }, - } - - return nil - }) - return err -} diff --git a/pkg/component/etcd/service/service_peer.go b/pkg/component/etcd/service/service_peer.go deleted file mode 100644 index caf35d0c0..000000000 --- a/pkg/component/etcd/service/service_peer.go +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "context" - - "github.com/gardener/gardener/pkg/controllerutils" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func (c *component) syncPeerService(ctx context.Context, svc *corev1.Service) error { - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, svc, func() error { - svc.Labels = c.values.Labels - svc.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - svc.Spec.Type = corev1.ServiceTypeClusterIP - svc.Spec.ClusterIP = corev1.ClusterIPNone - svc.Spec.SessionAffinity = corev1.ServiceAffinityNone - svc.Spec.Selector = c.values.SelectorLabels - svc.Spec.PublishNotReadyAddresses = true - svc.Spec.Ports = []corev1.ServicePort{ - { - Name: "peer", - Protocol: corev1.ProtocolTCP, - Port: c.values.PeerPort, - TargetPort: intstr.FromInt(int(c.values.PeerPort)), - }, - } - - return nil - }) - return err -} diff --git a/pkg/component/etcd/service/service_suite_test.go b/pkg/component/etcd/service/service_suite_test.go deleted file mode 100644 index d82edcf21..000000000 --- a/pkg/component/etcd/service/service_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestService(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Service Component Suite") -} diff --git a/pkg/component/etcd/service/service_test.go b/pkg/component/etcd/service/service_test.go deleted file mode 100644 index fcb081dbe..000000000 --- a/pkg/component/etcd/service/service_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service_test - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - . "github.com/gardener/etcd-druid/pkg/component/etcd/service" - "github.com/gardener/etcd-druid/pkg/utils" - - "github.com/gardener/gardener/pkg/component" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var _ = Describe("Service", func() { - var ( - ctx context.Context - cl client.Client - - etcd *druidv1alpha1.Etcd - backupPort, clientPort, serverPort int32 - namespace string - name string - uid types.UID - selectors map[string]string - services []*corev1.Service - - values Values - serviceDeployer component.Deployer - ) - - BeforeEach(func() { - ctx = context.Background() - namespace = "default" - cl = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - - backupPort = 1111 - clientPort = 2222 - serverPort = 3333 - - selectors = map[string]string{ - "foo": "bar", - "baz": "qux", - } - - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: uid, - }, - Spec: druidv1alpha1.EtcdSpec{ - Selector: metav1.SetAsLabelSelector(selectors), - Backup: druidv1alpha1.BackupSpec{ - Port: pointer.Int32(backupPort), - }, - Etcd: druidv1alpha1.EtcdConfig{ - ClientPort: pointer.Int32(clientPort), - ServerPort: pointer.Int32(serverPort), - }, - }, - } - - values = GenerateValues(etcd) - - services = []*corev1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: values.ClientServiceName, - Namespace: namespace, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: values.PeerServiceName, - Namespace: namespace, - }, - }, - } - - serviceDeployer = New(cl, namespace, values) - }) - - Describe("#Deploy", func() { - Context("when services do not exist", func() { - It("should create the service successfully", func() { - Expect(serviceDeployer.Deploy(ctx)).To(Succeed()) - - svc := &corev1.Service{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.ClientServiceName), svc)).To(Succeed()) - checkClientService(svc, values) - - Expect(cl.Get(ctx, kutil.Key(namespace, values.PeerServiceName), svc)).To(Succeed()) - checkPeerService(svc, values) - }) - }) - - Context("when services exist", func() { - It("should update the service successfully", func() { - for _, svc := range services { - Expect(cl.Create(ctx, svc)).To(Succeed()) - } - - Expect(serviceDeployer.Deploy(ctx)).To(Succeed()) - - svc := &corev1.Service{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.ClientServiceName), svc)).To(Succeed()) - checkClientService(svc, values) - - Expect(cl.Get(ctx, kutil.Key(namespace, values.PeerServiceName), svc)).To(Succeed()) - checkPeerService(svc, values) - }) - }) - }) - - Describe("#Destroy", func() { - Context("when services do not exist", func() { - It("should destroy successfully", func() { - Expect(serviceDeployer.Destroy(ctx)).To(Succeed()) - for _, svc := range services { - Expect(cl.Get(ctx, client.ObjectKeyFromObject(svc), &corev1.Service{})).To(BeNotFoundError()) - } - }) - }) - - Context("when services exist", func() { - It("should destroy successfully", func() { - for _, svc := range services { - Expect(cl.Create(ctx, svc)).To(Succeed()) - } - - Expect(serviceDeployer.Destroy(ctx)).To(Succeed()) - - for _, svc := range services { - Expect(cl.Get(ctx, kutil.Key(namespace, svc.Name), &corev1.Service{})).To(BeNotFoundError()) - } - }) - }) - }) -}) - -func checkClientService(svc *corev1.Service, values Values) { - Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{values.OwnerReference})) - Expect(svc.Labels).To(Equal(utils.MergeStringMaps(values.Labels, values.ClientServiceLabels))) - Expect(svc.Spec.Selector).To(Equal(values.SelectorLabels)) - Expect(svc.Spec.Type).To(Equal(corev1.ServiceType("ClusterIP"))) - Expect(svc.Spec.Ports).To(ConsistOf( - Equal(corev1.ServicePort{ - Name: "client", - Protocol: corev1.ProtocolTCP, - Port: values.ClientPort, - TargetPort: intstr.FromInt(int(values.ClientPort)), - }), - Equal(corev1.ServicePort{ - Name: "server", - Protocol: corev1.ProtocolTCP, - Port: values.PeerPort, - TargetPort: intstr.FromInt(int(values.PeerPort)), - }), - Equal(corev1.ServicePort{ - Name: "backuprestore", - Protocol: corev1.ProtocolTCP, - Port: values.BackupPort, - TargetPort: intstr.FromInt(int(values.BackupPort)), - }), - )) -} - -func checkPeerService(svc *corev1.Service, values Values) { - Expect(svc.OwnerReferences).To(Equal([]metav1.OwnerReference{values.OwnerReference})) - Expect(svc.Labels).To(Equal(values.Labels)) - Expect(svc.Spec.PublishNotReadyAddresses).To(BeTrue()) - Expect(svc.Spec.Type).To(Equal(corev1.ServiceType("ClusterIP"))) - Expect(svc.Spec.ClusterIP).To(Equal(("None"))) - Expect(svc.Spec.Selector).To(Equal(values.SelectorLabels)) - Expect(svc.Spec.Ports).To(ConsistOf( - Equal(corev1.ServicePort{ - Name: "peer", - Protocol: corev1.ProtocolTCP, - Port: values.PeerPort, - TargetPort: intstr.FromInt(int(values.PeerPort)), - }), - )) -} diff --git a/pkg/component/etcd/service/values.go b/pkg/component/etcd/service/values.go deleted file mode 100644 index 59c48ed33..000000000 --- a/pkg/component/etcd/service/values.go +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Values contains the values necessary for creating ETCD services. -type Values struct { - // BackupPort is the port exposed by the etcd-backup-restore side-car. - BackupPort int32 - // ClientPort is the port exposed by etcd for client communication. - ClientPort int32 - // ClientServiceName is the name of the service responsible for client traffic. - ClientServiceName string - // ClientAnnotations are the annotations to be added to the client service - ClientServiceAnnotations map[string]string - // ClientServiceLabels are the labels to be added to the client service - ClientServiceLabels map[string]string - // Labels are the service labels. - Labels map[string]string - // PeerServiceName is the name of the service responsible for peer traffic. - PeerServiceName string - // PeerPort is the port used for etcd peer communication. - PeerPort int32 - // OwnerReference is the OwnerReference for the ETCD services. - OwnerReference metav1.OwnerReference - // SelectorLabels are the labels to be used in the Service.spec selector - SelectorLabels map[string]string -} diff --git a/pkg/component/etcd/service/values_helper.go b/pkg/component/etcd/service/values_helper.go deleted file mode 100644 index 3c50e8be4..000000000 --- a/pkg/component/etcd/service/values_helper.go +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service - -import ( - "k8s.io/utils/pointer" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -) - -const ( - defaultBackupPort = 8080 - defaultClientPort = 2379 - defaultServerPort = 2380 -) - -// GenerateValues generates `service.Values` for the service component with the given `etcd` object. -func GenerateValues(etcd *druidv1alpha1.Etcd) Values { - var clientServiceAnnotations map[string]string - if etcd.Spec.Etcd.ClientService != nil { - clientServiceAnnotations = etcd.Spec.Etcd.ClientService.Annotations - } - var clientServiceLabels map[string]string - if etcd.Spec.Etcd.ClientService != nil { - clientServiceLabels = etcd.Spec.Etcd.ClientService.Labels - } - - return Values{ - BackupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, defaultBackupPort), - ClientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, defaultClientPort), - ClientServiceName: etcd.GetClientServiceName(), - ClientServiceAnnotations: clientServiceAnnotations, - ClientServiceLabels: clientServiceLabels, - SelectorLabels: etcd.GetDefaultLabels(), - Labels: etcd.GetDefaultLabels(), - PeerServiceName: etcd.GetPeerServiceName(), - PeerPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort), - OwnerReference: etcd.GetAsOwnerReference(), - } -} diff --git a/pkg/component/etcd/service/values_helper_test.go b/pkg/component/etcd/service/values_helper_test.go deleted file mode 100644 index 8da9b2c1a..000000000 --- a/pkg/component/etcd/service/values_helper_test.go +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package service_test - -import ( - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/component/etcd/service" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" -) - -var _ = Describe("#GenerateValues", func() { - var ( - etcd *druidv1alpha1.Etcd - - labels map[string]string - clientServiceAnnotations map[string]string - clientServiceLabels map[string]string - backupPort, clientPort, peerPort *int32 - expectedLabels = map[string]string{ - "instance": "etcd", - "name": "etcd", - } - ) - - JustBeforeEach(func() { - labels = map[string]string{ - "foo": "bar", - } - - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd", - Namespace: "default", - UID: "123", - }, - Spec: druidv1alpha1.EtcdSpec{ - Labels: labels, - Selector: metav1.SetAsLabelSelector(map[string]string{"baz": "qax"}), - Backup: druidv1alpha1.BackupSpec{ - Port: backupPort, - }, - Etcd: druidv1alpha1.EtcdConfig{ - ClientPort: clientPort, - ServerPort: peerPort, - ClientService: &druidv1alpha1.ClientService{ - Annotations: clientServiceAnnotations, - Labels: clientServiceLabels, - }, - }, - }, - } - }) - - Context("when ports are specified", func() { - BeforeEach(func() { - backupPort = pointer.Int32(1111) - clientPort = pointer.Int32(2222) - peerPort = pointer.Int32(3333) - }) - - It("should generate values correctly", func() { - values := GenerateValues(etcd) - - Expect(values).To(MatchFields(IgnoreExtras, Fields{ - "BackupPort": Equal(*etcd.Spec.Backup.Port), - "ClientPort": Equal(*etcd.Spec.Etcd.ClientPort), - "ClientServiceName": Equal(fmt.Sprintf("%s-client", etcd.Name)), - "Labels": Equal(expectedLabels), - "PeerServiceName": Equal(fmt.Sprintf("%s-peer", etcd.Name)), - "PeerPort": Equal(*etcd.Spec.Etcd.ServerPort), - })) - }) - }) - - Context("when ports aren't specified", func() { - BeforeEach(func() { - backupPort = nil - clientPort = nil - peerPort = nil - }) - - It("should generate values correctly", func() { - values := GenerateValues(etcd) - - Expect(values).To(MatchFields(IgnoreExtras, Fields{ - "BackupPort": Equal(int32(8080)), - "ClientPort": Equal(int32(2379)), - "ClientServiceName": Equal(fmt.Sprintf("%s-client", etcd.Name)), - "Labels": Equal(expectedLabels), - "PeerServiceName": Equal(fmt.Sprintf("%s-peer", etcd.Name)), - "PeerPort": Equal(int32(2380)), - "SelectorLabels": Equal(expectedLabels), - "OwnerReference": Equal(etcd.GetAsOwnerReference()), - })) - }) - }) - Context("when client service annotations are specified", func() { - BeforeEach(func() { - backupPort = nil - clientPort = nil - peerPort = nil - clientServiceAnnotations = map[string]string{ - "foo1": "bar1", - "foo2": "bar2", - } - }) - - It("should generate values correctly", func() { - values := GenerateValues(etcd) - - Expect(values).To(MatchFields(IgnoreExtras, Fields{ - "BackupPort": Equal(int32(8080)), - "ClientPort": Equal(int32(2379)), - "ClientServiceName": Equal(fmt.Sprintf("%s-client", etcd.Name)), - "Labels": Equal(expectedLabels), - "PeerServiceName": Equal(fmt.Sprintf("%s-peer", etcd.Name)), - "PeerPort": Equal(int32(2380)), - "ClientServiceAnnotations": Equal(clientServiceAnnotations), - "SelectorLabels": Equal(expectedLabels), - "OwnerReference": Equal(etcd.GetAsOwnerReference()), - })) - }) - }) - Context("when client service annotations are not specified", func() { - BeforeEach(func() { - backupPort = nil - clientPort = nil - peerPort = nil - clientServiceAnnotations = nil - }) - - It("should generate values correctly", func() { - values := GenerateValues(etcd) - - Expect(values).To(MatchFields(IgnoreExtras, Fields{ - "BackupPort": Equal(int32(8080)), - "ClientPort": Equal(int32(2379)), - "ClientServiceName": Equal(fmt.Sprintf("%s-client", etcd.Name)), - "Labels": Equal(expectedLabels), - "PeerServiceName": Equal(fmt.Sprintf("%s-peer", etcd.Name)), - "PeerPort": Equal(int32(2380)), - "ClientServiceAnnotations": Equal(clientServiceAnnotations), - "SelectorLabels": Equal(expectedLabels), - "OwnerReference": Equal(etcd.GetAsOwnerReference()), - })) - }) - }) - Context("when client service labels are specified", func() { - BeforeEach(func() { - backupPort = nil - clientPort = nil - peerPort = nil - clientServiceLabels = map[string]string{ - "foo1": "bar1", - "foo2": "bar2", - } - }) - - It("should generate values correctly", func() { - values := GenerateValues(etcd) - - Expect(values).To(MatchFields(IgnoreExtras, Fields{ - "BackupPort": Equal(int32(8080)), - "ClientPort": Equal(int32(2379)), - "ClientServiceName": Equal(fmt.Sprintf("%s-client", etcd.Name)), - "Labels": Equal(expectedLabels), - "PeerServiceName": Equal(fmt.Sprintf("%s-peer", etcd.Name)), - "PeerPort": Equal(int32(2380)), - "ClientServiceLabels": Equal(clientServiceLabels), - "SelectorLabels": Equal(expectedLabels), - "OwnerReference": Equal(etcd.GetAsOwnerReference()), - })) - }) - }) - Context("when client service labels are not specified", func() { - BeforeEach(func() { - backupPort = nil - clientPort = nil - peerPort = nil - clientServiceLabels = nil - }) - - It("should generate values correctly", func() { - values := GenerateValues(etcd) - - Expect(values).To(MatchFields(IgnoreExtras, Fields{ - "BackupPort": Equal(int32(8080)), - "ClientPort": Equal(int32(2379)), - "ClientServiceName": Equal(fmt.Sprintf("%s-client", etcd.Name)), - "Labels": Equal(expectedLabels), - "PeerServiceName": Equal(fmt.Sprintf("%s-peer", etcd.Name)), - "PeerPort": Equal(int32(2380)), - "ClientServiceLabels": Equal(clientServiceLabels), - "SelectorLabels": Equal(expectedLabels), - "OwnerReference": Equal(etcd.GetAsOwnerReference()), - })) - }) - }) -}) diff --git a/pkg/component/etcd/serviceaccount/serviceaccount.go b/pkg/component/etcd/serviceaccount/serviceaccount.go deleted file mode 100644 index 875c01a1f..000000000 --- a/pkg/component/etcd/serviceaccount/serviceaccount.go +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package serviceaccount - -import ( - "context" - - "github.com/gardener/gardener/pkg/controllerutils" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type component struct { - client client.Client - values *Values -} - -func (c component) Deploy(ctx context.Context) error { - serviceAccount := c.emptyServiceAccount() - _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, serviceAccount, func() error { - serviceAccount.Name = c.values.Name - serviceAccount.Namespace = c.values.Namespace - serviceAccount.Labels = c.values.Labels - serviceAccount.OwnerReferences = []metav1.OwnerReference{c.values.OwnerReference} - serviceAccount.AutomountServiceAccountToken = pointer.Bool(!c.values.DisableAutomount) - return nil - }) - return err -} - -func (c component) Destroy(ctx context.Context) error { - return client.IgnoreNotFound(c.client.Delete(ctx, c.emptyServiceAccount())) -} - -func (c component) emptyServiceAccount() *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.Name, - Namespace: c.values.Namespace, - }, - } -} - -// New creates a new service account deployer instance. -func New(c client.Client, value *Values) gardenercomponent.Deployer { - return &component{ - client: c, - values: value, - } -} diff --git a/pkg/component/etcd/serviceaccount/serviceaccount_suite_test.go b/pkg/component/etcd/serviceaccount/serviceaccount_suite_test.go deleted file mode 100644 index 0a79d3f65..000000000 --- a/pkg/component/etcd/serviceaccount/serviceaccount_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package serviceaccount - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestServiceAccount(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "ServiceAccount Component Suite") -} diff --git a/pkg/component/etcd/serviceaccount/serviceaccount_test.go b/pkg/component/etcd/serviceaccount/serviceaccount_test.go deleted file mode 100644 index 1e6567903..000000000 --- a/pkg/component/etcd/serviceaccount/serviceaccount_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package serviceaccount - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - gardenercomponent "github.com/gardener/gardener/pkg/component" - - . "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var _ = Describe("ServiceAccount Component", Ordered, func() { - var ( - saComponent gardenercomponent.Deployer - ctx context.Context - c client.Client - values *Values - ) - - Context("#Deploy", func() { - - BeforeEach(func() { - ctx = context.Background() - c = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - - values = &Values{ - Name: "test-service-account", - Namespace: "test-namespace", - Labels: map[string]string{ - "foo": "bar", - }, - DisableAutomount: true, - OwnerReference: metav1.OwnerReference{ - APIVersion: druidv1alpha1.GroupVersion.String(), - Kind: "Etcd", - Name: "test-etcd", - UID: "123-456-789", - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - }, - } - saComponent = New(c, values) - err := saComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - err := saComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - - It("should create the ServiceAccount with the expected values", func() { - By("verifying that the ServiceAccount is created on the K8s cluster as expected") - created := &corev1.ServiceAccount{} - err := c.Get(ctx, getServiceAccountKeyFromValue(values), created) - Expect(err).NotTo(HaveOccurred()) - verifyServicAccountValues(created, values) - }) - - It("should update the ServiceAccount with the expected values", func() { - By("updating the ServiceAccount") - values.Labels["new"] = "label" - err := saComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that the ServiceAccount is updated on the K8s cluster as expected") - updated := &corev1.ServiceAccount{} - err = c.Get(ctx, getServiceAccountKeyFromValue(values), updated) - Expect(err).NotTo(HaveOccurred()) - verifyServicAccountValues(updated, values) - }) - - It("should not return an error when there is nothing to update the ServiceAccount", func() { - err := saComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - updated := &corev1.ServiceAccount{} - err = c.Get(ctx, getServiceAccountKeyFromValue(values), updated) - Expect(err).NotTo(HaveOccurred()) - verifyServicAccountValues(updated, values) - }) - - It("should return an error when the update fails", func() { - values.Name = "" - err := saComponent.Deploy(ctx) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Required value: name is required")) - }) - }) - - Context("#Destroy", func() { - BeforeEach(func() { - ctx = context.Background() - c = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - - values = &Values{ - Name: "test-service-account", - Namespace: "test-namespace", - Labels: map[string]string{ - "foo": "bar", - }, - DisableAutomount: true, - OwnerReference: metav1.OwnerReference{ - APIVersion: druidv1alpha1.GroupVersion.String(), - Kind: "Etcd", - Name: "test-etcd", - UID: "123-456-789", - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - }, - } - saComponent = New(c, values) - err := saComponent.Deploy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - - It("should delete the ServiceAccount", func() { - By("deleting the ServiceAccount") - err := saComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that the ServiceAccount is deleted from the K8s cluster as expected") - sa := &corev1.ServiceAccount{} - Expect(c.Get(ctx, getServiceAccountKeyFromValue(values), sa)).To(BeNotFoundError()) - }) - - It("should not return an error when there is nothing to delete", func() { - By("deleting the ServiceAccount") - err := saComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - - By("verifying that attempting to delete the ServiceAccount again returns no error") - err = saComponent.Destroy(ctx) - Expect(err).NotTo(HaveOccurred()) - }) - }) -}) - -func getServiceAccountKeyFromValue(value *Values) types.NamespacedName { - return client.ObjectKey{Name: value.Name, Namespace: value.Namespace} -} - -func verifyServicAccountValues(expected *corev1.ServiceAccount, values *Values) { - Expect(expected.Name).To(Equal(values.Name)) - Expect(expected.Labels).Should(Equal(values.Labels)) - Expect(expected.Namespace).To(Equal(values.Namespace)) - Expect(expected.OwnerReferences).To(Equal([]metav1.OwnerReference{values.OwnerReference})) - Expect(expected.AutomountServiceAccountToken).To(Equal(pointer.Bool(!values.DisableAutomount))) -} diff --git a/pkg/component/etcd/serviceaccount/values.go b/pkg/component/etcd/serviceaccount/values.go deleted file mode 100644 index f831ab43c..000000000 --- a/pkg/component/etcd/serviceaccount/values.go +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package serviceaccount - -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - -// Values defines the fields used to create a ServiceAccount for Etcd. -type Values struct { - // Name is the name of the ServiceAccount. - Name string - // Namespace is the namespace of the ServiceAccount. - Namespace string - // OwnerReference is the OwnerReference for the ServiceAccount. - OwnerReference metav1.OwnerReference - // Labels are the labels to apply to the ServiceAccount. - Labels map[string]string - // DisableAutomount defines the AutomountServiceAccountToken of the ServiceAccount. - DisableAutomount bool -} diff --git a/pkg/component/etcd/serviceaccount/values_helper.go b/pkg/component/etcd/serviceaccount/values_helper.go deleted file mode 100644 index bd7fe39f5..000000000 --- a/pkg/component/etcd/serviceaccount/values_helper.go +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package serviceaccount - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -) - -// GenerateValues generates `serviceaccount.Values` for the serviceaccount component for the given `etcd` object. -func GenerateValues(etcd *druidv1alpha1.Etcd, disableEtcdServiceAccountAutomount bool) *Values { - return &Values{ - Name: etcd.GetServiceAccountName(), - Namespace: etcd.Namespace, - Labels: etcd.GetDefaultLabels(), - OwnerReference: etcd.GetAsOwnerReference(), - DisableAutomount: disableEtcdServiceAccountAutomount, - } -} diff --git a/pkg/component/etcd/serviceaccount/values_helper_test.go b/pkg/component/etcd/serviceaccount/values_helper_test.go deleted file mode 100644 index 9bb928c60..000000000 --- a/pkg/component/etcd/serviceaccount/values_helper_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package serviceaccount_test - -import ( - "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/component/etcd/serviceaccount" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var _ = Describe("ServiceAccount", func() { - var ( - etcd *v1alpha1.Etcd - expected *serviceaccount.Values - ) - - BeforeEach(func() { - etcd = &v1alpha1.Etcd{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "Etcd", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd-name", - Namespace: "etcd-namespace", - UID: "etcd-uid", - }, - Spec: v1alpha1.EtcdSpec{ - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - } - expected = &serviceaccount.Values{ - Name: etcd.GetServiceAccountName(), - Namespace: etcd.Namespace, - Labels: map[string]string{ - "name": "etcd", - "instance": etcd.Name, - }, - OwnerReference: etcd.GetAsOwnerReference(), - - DisableAutomount: true, - } - }) - Context("Generate Values", func() { - It("should generate correct values with automount disabled", func() { - values := serviceaccount.GenerateValues(etcd, true) - Expect(values).To(Equal(expected)) - }) - - It("should generate correct values with automount enabled", func() { - values := serviceaccount.GenerateValues(etcd, false) - expected.DisableAutomount = false - Expect(values).To(Equal(expected)) - }) - }) -}) diff --git a/pkg/component/etcd/statefulset/statefulset.go b/pkg/component/etcd/statefulset/statefulset.go deleted file mode 100644 index 93eb5684d..000000000 --- a/pkg/component/etcd/statefulset/statefulset.go +++ /dev/null @@ -1,948 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package statefulset - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" - - gardenercomponent "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/utils/flow" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/gardener/gardener/pkg/utils/retry" - gardenerretry "github.com/gardener/gardener/pkg/utils/retry" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/component-base/featuregate" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Interface contains functions for a StatefulSet deployer. -type Interface interface { - gardenercomponent.DeployWaiter - // Get gets the etcd StatefulSet. - Get(context.Context) (*appsv1.StatefulSet, error) -} - -type component struct { - client client.Client - logger logr.Logger - values Values - featureGates map[featuregate.Feature]bool -} - -// New creates a new StatefulSet deployer instance. -func New(c client.Client, logger logr.Logger, values Values, featureGates map[featuregate.Feature]bool) Interface { - objectLogger := logger.WithValues("sts", client.ObjectKey{Name: values.Name, Namespace: values.Namespace}) - - return &component{ - client: c, - logger: objectLogger, - values: values, - featureGates: featureGates, - } -} - -// Destroy deletes the StatefulSet -func (c *component) Destroy(ctx context.Context) error { - sts := c.emptyStatefulset() - return client.IgnoreNotFound(c.client.Delete(ctx, sts)) -} - -// Deploy executes a deploy-flow to ensure that the StatefulSet is synchronized correctly -func (c *component) Deploy(ctx context.Context) error { - deployFlow, err := c.createDeployFlow(ctx) - if err != nil { - return err - } - return deployFlow.Run(ctx, flow.Opts{}) -} - -// Get retrieves the existing StatefulSet -func (c *component) Get(ctx context.Context) (*appsv1.StatefulSet, error) { - sts := c.emptyStatefulset() - if err := c.client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { - return nil, err - } - return sts, nil -} - -const ( - // defaultInterval is the default interval for retry operations. - defaultInterval = 5 * time.Second - // defaultTimeout is the default timeout for retry operations. - defaultTimeout = 90 * time.Second - - // ScaleToMultiNodeAnnotationKey is used to represent scale-up annotation. - ScaleToMultiNodeAnnotationKey = "gardener.cloud/scaled-to-multi-node" -) - -// Wait waits for the deployment of the StatefulSet to finish -func (c *component) Wait(ctx context.Context) error { - sts := c.emptyStatefulset() - err := c.waitDeploy(ctx, sts, c.values.Replicas, defaultTimeout) - if err != nil { - if getErr := c.client.Get(ctx, client.ObjectKeyFromObject(sts), sts); getErr != nil { - return err - } - messages, errorPVC, err2 := c.fetchPVCEventsForStatefulset(ctx, sts) - if err2 != nil { - c.logger.Error(err2, "Error while fetching events for depending PVCs for statefulset", - "namespace", sts.Namespace, "statefulsetName", sts.Name, "pvcName", *errorPVC) - // don't expose this error since fetching events is the best effort - // and shouldn't be confused with the actual error - return err - } - if messages != "" { - return fmt.Errorf("%w\n\n%s", err, messages) - } - } - - return err -} - -func (c *component) getLatestPodCreationTime(ctx context.Context, sts *appsv1.StatefulSet) (time.Time, error) { - pods := corev1.PodList{} - if err := c.client.List(ctx, &pods, client.InNamespace(sts.Namespace), client.MatchingLabels(sts.Spec.Template.Labels)); err != nil { - return time.Time{}, err - } - var recentCreationTime time.Time - for _, pod := range pods.Items { - if recentCreationTime.Before(pod.CreationTimestamp.Time) { - recentCreationTime = pod.CreationTimestamp.Time - } - } - return recentCreationTime, nil -} - -func (c *component) waitUntilPodsReady(ctx context.Context, originalSts *appsv1.StatefulSet, podDeletionTime time.Time, interval time.Duration, timeout time.Duration) error { - sts := appsv1.StatefulSet{} - return gardenerretry.UntilTimeout(ctx, interval, timeout, func(ctx context.Context) (bool, error) { - if err := c.client.Get(ctx, client.ObjectKeyFromObject(originalSts), &sts); err != nil { - if apierrors.IsNotFound(err) { - return gardenerretry.MinorError(err) - } - return gardenerretry.SevereError(err) - } - if sts.Status.ReadyReplicas < *sts.Spec.Replicas { - return gardenerretry.MinorError(fmt.Errorf("only %d out of %d replicas are ready", sts.Status.ReadyReplicas, sts.Spec.Replicas)) - } - recentPodCreationTime, err := c.getLatestPodCreationTime(ctx, &sts) - if err != nil { - return gardenerretry.MinorError(fmt.Errorf("failed to get most recent pod creation timestamp: %w", err)) - } - if recentPodCreationTime.Before(podDeletionTime) { - return gardenerretry.MinorError(fmt.Errorf("most recent pod creation time %v is still before the %v time when the pods were deleted", recentPodCreationTime, podDeletionTime)) - } - return gardenerretry.Ok() - }) -} - -func (c *component) waitDeploy(ctx context.Context, originalSts *appsv1.StatefulSet, replicas int32, timeout time.Duration) error { - updatedSts := appsv1.StatefulSet{} - return gardenerretry.UntilTimeout(ctx, defaultInterval, timeout, func(ctx context.Context) (bool, error) { - if err := c.client.Get(ctx, client.ObjectKeyFromObject(originalSts), &updatedSts); err != nil { - if apierrors.IsNotFound(err) { - return gardenerretry.MinorError(err) - } - return gardenerretry.SevereError(err) - } - if updatedSts.Generation < originalSts.Generation { - return gardenerretry.MinorError(fmt.Errorf("statefulset generation has not yet been updated in the cache")) - } - if ready, reason := utils.IsStatefulSetReady(replicas, &updatedSts); !ready { - return gardenerretry.MinorError(fmt.Errorf(reason)) - } - return gardenerretry.Ok() - }) -} - -// WaitCleanup waits for the deletion of the StatefulSet to complete -func (c *component) WaitCleanup(ctx context.Context) error { - return gardenerretry.UntilTimeout(ctx, defaultInterval, defaultTimeout, func(ctx context.Context) (done bool, err error) { - sts := c.emptyStatefulset() - err = c.client.Get(ctx, client.ObjectKeyFromObject(sts), sts) - switch { - case apierrors.IsNotFound(err): - return retry.Ok() - case err == nil: - // StatefulSet is still available, so we should retry. - return false, nil - default: - return retry.SevereError(err) - } - }) -} - -func (c *component) createDeployFlow(ctx context.Context) (*flow.Flow, error) { - var ( - sts *appsv1.StatefulSet - err error - ) - sts, err = c.getExistingSts(ctx) - if err != nil { - return nil, err - } - - flowName := fmt.Sprintf("(etcd: %s) Deploy Flow for StatefulSet %s for Namespace: %s", getOwnerReferenceNameWithUID(c.values.OwnerReference), c.values.Name, c.values.Namespace) - g := flow.NewGraph(flowName) - - var taskID *flow.TaskID - if sts != nil { - taskID = c.addTasksForPeerUrlTLSChangedToEnabled(g, sts) - if taskID == nil { - // if sts recreation tasks for peer url tls have already been added then there is no need to additionally add tasks to explicitly handle immutable field updates. - taskID = c.addImmutableFieldUpdateTask(g, sts) - } - } - c.addCreateOrPatchTask(g, sts, taskID) - - return g.Compile(), nil -} - -// addTasksForPeerUrlTLSChangedToEnabled adds tasks to the deployment flow in case the peer url tls has been changed to `enabled`. -// To ensure that the tls enablement of peer url is properly reflected in etcd, the existing etcd StatefulSet pods should be restarted twice. Assume -// that the current state of etcd is that peer url is not TLS enabled. First restart pushes a new configuration which contains -// PeerUrlTLS configuration. etcd-backup-restore will update the member peer url. This will result in the change of the peer url in the etcd db file, -// but it will not reflect in the already running etcd container. Ideally a restart of an etcd container would have been sufficient but currently k8s -// does not expose an API to force restart a single container within a pod. Therefore, we need to restart the StatefulSet pod(s) once again. When the pod(s) is -// restarted the second time it will now start etcd with the correct peer url which will be TLS enabled. -// To achieve 2 restarts following is done: -// 1. An update is made to the spec mounting the peer URL TLS secrets. This will cause a rolling update of the existing pod. -// 2. Once the update is successfully completed, then we delete StatefulSet pods, causing a restart by the StatefulSet controller. -// NOTE: The need to restart etcd pods twice will change in the future. -func (c *component) addTasksForPeerUrlTLSChangedToEnabled(g *flow.Graph, sts *appsv1.StatefulSet) *flow.TaskID { - var existingStsReplicas int32 - if sts.Spec.Replicas != nil { - existingStsReplicas = *sts.Spec.Replicas - } - - if c.values.PeerTLSChangedToEnabled { - updateStsOpName := "(update-sts-spec): update Peer TLS secret mount" - updateTaskID := g.Add(flow.Task{ - Name: updateStsOpName, - Fn: func(ctx context.Context) error { - return c.updateAndWait(ctx, updateStsOpName, sts, existingStsReplicas) - }, - Dependencies: nil, - }) - c.logger.Info("adding task to deploy flow", "name", updateStsOpName, "ID", updateTaskID) - - waitForLeaseUpdateOpName := "(wait-lease-update): Wait for lease to be updated with peer TLS" - waitLeaseUpdateID := g.Add(flow.Task{ - Name: waitForLeaseUpdateOpName, - Fn: func(ctx context.Context) error { - return c.waitUntilTLSEnabled(ctx, 3*time.Minute) - }, - Dependencies: flow.NewTaskIDs(updateTaskID), - }) - c.logger.Info("adding task to deploy flow", "name", updateStsOpName, "ID", waitLeaseUpdateID) - - deleteAllStsPodsOpName := "(delete-sts-pods): deleting all sts pods" - deleteAllStsPodsTaskID := g.Add(flow.Task{ - Name: deleteAllStsPodsOpName, - Fn: func(ctx context.Context) error { - return c.deleteAllStsPods(ctx, deleteAllStsPodsOpName, sts) - }, - Dependencies: flow.NewTaskIDs(waitLeaseUpdateID), - }) - - c.logger.Info("adding task to deploy flow", "name", deleteAllStsPodsOpName, "ID", deleteAllStsPodsTaskID) - - return &deleteAllStsPodsTaskID - } - return nil -} - -func (c *component) addImmutableFieldUpdateTask(g *flow.Graph, sts *appsv1.StatefulSet) *flow.TaskID { - if sts.Generation > 1 && immutableFieldUpdate(sts, c.values) { - opName := "delete sts due to immutable field update" - taskID := g.Add(flow.Task{ - Name: opName, - Fn: func(ctx context.Context) error { return c.destroyAndWait(ctx, opName) }, - Dependencies: nil, - }) - c.logger.Info("added delete StatefulSet task to deploy flow due to immutable field update task", "namespace", c.values.Namespace, "name", c.values.Name, "etcdUID", getOwnerReferenceNameWithUID(c.values.OwnerReference)) - return &taskID - } - return nil -} - -func (c *component) addCreateOrPatchTask(g *flow.Graph, originalSts *appsv1.StatefulSet, taskIDDependency *flow.TaskID) { - var ( - dependencies flow.TaskIDs - ) - if taskIDDependency != nil { - dependencies = flow.NewTaskIDs(taskIDDependency) - } - - taskID := g.Add(flow.Task{ - Name: "sync StatefulSet task", - Fn: func(ctx context.Context) error { - c.logger.Info("createOrPatch sts", "namespace", c.values.Namespace, "name", c.values.Name, "replicas", c.values.Replicas) - var ( - sts = originalSts - err error - ) - if taskIDDependency != nil { - sts, err = c.getExistingSts(ctx) - if err != nil { - return err - } - } - if sts == nil { - sts = c.emptyStatefulset() - } - return c.createOrPatch(ctx, sts, c.values.Replicas, false) - }, - Dependencies: dependencies, - }) - c.logger.Info("added createOrPatch StatefulSet task to the deploy flow", "taskID", taskID, "namespace", c.values.Namespace, "etcdUID", getOwnerReferenceNameWithUID(c.values.OwnerReference), "StatefulSetName", c.values.Name, "replicas", c.values.Replicas) -} - -func (c *component) getExistingSts(ctx context.Context) (*appsv1.StatefulSet, error) { - sts, err := c.Get(ctx) - if err != nil { - if apierrors.IsNotFound(err) { - return nil, nil - } - return nil, err - } - return sts, nil -} - -func (c *component) updateAndWait(ctx context.Context, opName string, sts *appsv1.StatefulSet, replicas int32) error { - c.logger.Info("Updating StatefulSet spec with Peer URL TLS mount", "namespace", c.values.Namespace, "name", c.values.Name, "operation", opName, "etcdUID", getOwnerReferenceNameWithUID(c.values.OwnerReference), "replicas", replicas) - return c.doCreateOrUpdate(ctx, opName, sts, replicas, true) -} - -func (c *component) waitUntilTLSEnabled(ctx context.Context, timeout time.Duration) error { - return gardenerretry.UntilTimeout(ctx, defaultInterval, timeout, func(ctx context.Context) (bool, error) { - tlsEnabled, err := utils.IsPeerURLTLSEnabled(ctx, c.client, c.values.Namespace, c.values.Name, c.logger) - if err != nil { - return gardenerretry.MinorError(err) - } - if !tlsEnabled { - return gardenerretry.MinorError(fmt.Errorf("TLS not yet enabled for etcd [name: %s, namespace: %s]", c.values.Name, c.values.Namespace)) - } - return gardenerretry.Ok() - }) -} - -func (c *component) deleteAllStsPods(ctx context.Context, opName string, sts *appsv1.StatefulSet) error { - replicas := sts.Spec.Replicas - c.logger.Info("Deleting all StatefulSet pods", "namespace", c.values.Namespace, "name", c.values.Name, "operation", opName, "etcdUID", getOwnerReferenceNameWithUID(c.values.OwnerReference), "replicas", replicas, "matching labels", sts.Spec.Template.Labels) - timeBeforeDeletion := time.Now() - - if err := c.client.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(sts.Namespace), client.MatchingLabels(sts.Spec.Template.Labels)); err != nil { - return err - } - - const timeout = 3 * time.Minute - const interval = 2 * time.Second - - c.logger.Info("waiting for StatefulSet pods to start again after delete", "namespace", c.values.Namespace, "name", c.values.Name, "operation", opName, "etcdUID", getOwnerReferenceNameWithUID(c.values.OwnerReference), "replicas", replicas) - return c.waitUntilPodsReady(ctx, sts, timeBeforeDeletion, interval, timeout) -} - -func (c *component) doCreateOrUpdate(ctx context.Context, opName string, sts *appsv1.StatefulSet, replicas int32, preserveObjectMetadata bool) error { - if err := c.createOrPatch(ctx, sts, replicas, preserveObjectMetadata); err != nil { - return err - } - c.logger.Info("waiting for StatefulSet replicas to be ready", "namespace", c.values.Namespace, "name", c.values.Name, "operation", opName, "target-replicas", replicas) - return c.waitDeploy(ctx, sts, replicas, defaultTimeout) -} - -func (c *component) destroyAndWait(ctx context.Context, opName string) error { - deleteAndWait := gardenercomponent.OpDestroyAndWait(c) - c.logger.Info("deleting sts", "namespace", c.values.Namespace, "name", c.values.Name, "operation", opName, "etcdUID", getOwnerReferenceNameWithUID(c.values.OwnerReference)) - return deleteAndWait.Destroy(ctx) -} - -func immutableFieldUpdate(sts *appsv1.StatefulSet, val Values) bool { - return sts.Spec.ServiceName != val.PeerServiceName || sts.Spec.PodManagementPolicy != appsv1.ParallelPodManagement -} - -func clusterScaledUpToMultiNode(val *Values, sts *appsv1.StatefulSet) bool { - if sts != nil && sts.Spec.Replicas != nil { - return (val.Replicas > 1 && *sts.Spec.Replicas == 1 && sts.Status.AvailableReplicas == 1) || - (metav1.HasAnnotation(sts.ObjectMeta, ScaleToMultiNodeAnnotationKey) && - (sts.Status.UpdatedReplicas < *sts.Spec.Replicas || sts.Status.AvailableReplicas < sts.Status.UpdatedReplicas)) - } - return val.Replicas > 1 && val.StatusReplicas == 1 -} - -func (c *component) createOrPatch(ctx context.Context, sts *appsv1.StatefulSet, replicas int32, preserveAnnotations bool) error { - mutatingFn := func() error { - var stsOriginal = sts.DeepCopy() - - podVolumes, err := getVolumes(ctx, c.client, c.logger, c.values) - if err != nil { - return err - } - sts.ObjectMeta = getObjectMeta(&c.values, sts, preserveAnnotations) - sts.Spec = appsv1.StatefulSetSpec{ - PodManagementPolicy: appsv1.ParallelPodManagement, - UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: appsv1.RollingUpdateStatefulSetStrategyType, - }, - Replicas: &replicas, - ServiceName: c.values.PeerServiceName, - Selector: &metav1.LabelSelector{ - MatchLabels: c.values.Labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: c.values.Annotations, - Labels: utils.MergeStringMaps(make(map[string]string), c.values.AdditionalPodLabels, c.values.Labels), - }, - Spec: corev1.PodSpec{ - HostAliases: []corev1.HostAlias{ - { - IP: "127.0.0.1", - Hostnames: []string{c.values.Name + "-local"}, - }, - }, - ServiceAccountName: c.values.ServiceAccountName, - Affinity: c.values.Affinity, - TopologySpreadConstraints: c.values.TopologySpreadConstraints, - Containers: []corev1.Container{ - { - Name: "etcd", - Image: c.values.EtcdImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: c.values.EtcdCommandArgs, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: getReadinessHandler(c.values), - InitialDelaySeconds: 15, - PeriodSeconds: 5, - FailureThreshold: 5, - }, - Ports: getEtcdPorts(c.values), - Resources: getEtcdResources(c.values), - Env: getEtcdEnvVars(c.values), - VolumeMounts: getEtcdVolumeMounts(c.values), - }, - { - Name: "backup-restore", - Image: c.values.BackupImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: c.values.EtcdBackupRestoreCommandArgs, - Ports: getBackupPorts(c.values), - Resources: getBackupResources(c.values), - VolumeMounts: getBackupRestoreVolumeMounts(c), - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_PTRACE", - }, - }, - }, - }, - }, - ShareProcessNamespace: pointer.Bool(true), - Volumes: podVolumes, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.VolumeClaimTemplateName, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: getStorageReq(c.values), - }, - }, - }, - } - if backupRestoreEnvVars, err := utils.GetBackupRestoreContainerEnvVars(c.values.BackupStore); err != nil { - return fmt.Errorf("unable to fetch env vars for backup-restore container: %v", err) - } else { - sts.Spec.Template.Spec.Containers[1].Env = backupRestoreEnvVars - } - if c.values.StorageClass != nil && *c.values.StorageClass != "" { - sts.Spec.VolumeClaimTemplates[0].Spec.StorageClassName = c.values.StorageClass - } - if c.values.PriorityClassName != nil { - sts.Spec.Template.Spec.PriorityClassName = *c.values.PriorityClassName - } - if c.values.UseEtcdWrapper { - // sections to add only when using etcd wrapper - // TODO: @aaronfern add this back to sts.Spec when UseEtcdWrapper becomes GA - sts.Spec.Template.Spec.InitContainers = []corev1.Container{ - { - Name: "change-permissions", - Image: c.values.InitContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{"chown -R 65532:65532 /var/etcd/data"}, - VolumeMounts: getEtcdVolumeMounts(c.values), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }, - } - if c.values.BackupStore != nil { - // Special container to change permissions of backup bucket folder to 65532 (nonroot) - // Only used with local provider - prov, _ := utils.StorageProviderFromInfraProvider(c.values.BackupStore.Provider) - if prov == utils.Local { - sts.Spec.Template.Spec.InitContainers = append(sts.Spec.Template.Spec.InitContainers, corev1.Container{ - Name: "change-backup-bucket-permissions", - Image: c.values.InitContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *c.values.BackupStore.Container)}, - VolumeMounts: getBackupRestoreVolumeMounts(c), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }) - } - } - sts.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ - RunAsGroup: pointer.Int64(65532), - RunAsNonRoot: pointer.Bool(true), - RunAsUser: pointer.Int64(65532), - } - } - - if stsOriginal.Generation > 0 { - // Keep immutable fields - sts.Spec.PodManagementPolicy = stsOriginal.Spec.PodManagementPolicy - sts.Spec.ServiceName = stsOriginal.Spec.ServiceName - } - return nil - } - - operationResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, sts, mutatingFn) - if err != nil { - return err - } - c.logger.Info("createOrPatch is completed", "namespace", sts.Namespace, "name", sts.Name, "operation-result", operationResult) - return nil -} - -// fetchPVCEventsForStatefulset fetches events for PVCs for a statefulset and return the events, -// as well as possible error and name of the PVC that caused the error -func (c *component) fetchPVCEventsForStatefulset(ctx context.Context, ss *appsv1.StatefulSet) (string, *string, error) { - pvcs := &corev1.PersistentVolumeClaimList{} - if err := c.client.List(ctx, pvcs, client.InNamespace(ss.GetNamespace())); err != nil { - return "", pointer.String(""), err - } - - var ( - pvcMessages string - volumeClaims = ss.Spec.VolumeClaimTemplates - ) - for _, volumeClaim := range volumeClaims { - for _, pvc := range pvcs.Items { - if !strings.HasPrefix(pvc.GetName(), fmt.Sprintf("%s-%s", volumeClaim.Name, ss.Name)) || pvc.Status.Phase == corev1.ClaimBound { - continue - } - messages, err := kutil.FetchEventMessages(ctx, c.client.Scheme(), c.client, &pvc, corev1.EventTypeWarning, 2) - if err != nil { - return "", &pvc.Name, err - } - if messages != "" { - pvcMessages += fmt.Sprintf("Warning for PVC %s:\n%s\n", pvc.Name, messages) - } - } - } - return pvcMessages, nil, nil -} - -func (c *component) emptyStatefulset() *appsv1.StatefulSet { - return &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.values.Name, - Namespace: c.values.Namespace, - }, - } -} - -func getObjectMeta(val *Values, sts *appsv1.StatefulSet, preserveAnnotations bool) metav1.ObjectMeta { - annotations := sts.Annotations - if !preserveAnnotations { - annotations = getStsAnnotations(val, sts) - } - - return metav1.ObjectMeta{ - Name: val.Name, - Namespace: val.Namespace, - Labels: val.Labels, - Annotations: annotations, - OwnerReferences: []metav1.OwnerReference{val.OwnerReference}, - } -} - -func getStsAnnotations(val *Values, sts *appsv1.StatefulSet) map[string]string { - annotations := utils.MergeStringMaps( - map[string]string{ - common.GardenerOwnedBy: fmt.Sprintf("%s/%s", val.Namespace, val.Name), - common.GardenerOwnerType: "etcd", - }, - val.Annotations, - ) - - if clusterScaledUpToMultiNode(val, sts) { - annotations[ScaleToMultiNodeAnnotationKey] = "" - } - return annotations -} - -func getEtcdPorts(val Values) []corev1.ContainerPort { - return []corev1.ContainerPort{ - { - Name: "server", - Protocol: "TCP", - ContainerPort: pointer.Int32Deref(val.ServerPort, defaultServerPort), - }, - { - Name: "client", - Protocol: "TCP", - ContainerPort: pointer.Int32Deref(val.ClientPort, defaultClientPort), - }, - } -} - -var defaultResourceRequirements = corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, -} - -func getEtcdResources(val Values) corev1.ResourceRequirements { - if val.EtcdResourceRequirements != nil { - return *val.EtcdResourceRequirements - } - - return defaultResourceRequirements -} - -func getEtcdEnvVars(val Values) []corev1.EnvVar { - protocol := "http" - if val.BackupTLS != nil { - protocol = "https" - } - - endpoint := fmt.Sprintf("%s://%s-local:%d", protocol, val.Name, pointer.Int32Deref(val.BackupPort, defaultBackupPort)) - - return []corev1.EnvVar{ - utils.GetEnvVarFromValue("ENABLE_TLS", strconv.FormatBool(val.BackupTLS != nil)), - utils.GetEnvVarFromValue("BACKUP_ENDPOINT", endpoint), - } -} - -func getEtcdVolumeMounts(val Values) []corev1.VolumeMount { - vms := []corev1.VolumeMount{ - { - Name: val.VolumeClaimTemplateName, - MountPath: "/var/etcd/data/", - }, - } - - vms = append(vms, getSecretVolumeMounts(val.ClientUrlTLS, val.PeerUrlTLS)...) - - return vms -} - -func getSecretVolumeMounts(clientUrlTLS, peerUrlTLS *druidv1alpha1.TLSConfig) []corev1.VolumeMount { - var vms []corev1.VolumeMount - - if clientUrlTLS != nil { - vms = append(vms, corev1.VolumeMount{ - Name: "client-url-ca-etcd", - MountPath: "/var/etcd/ssl/client/ca", - }, corev1.VolumeMount{ - Name: "client-url-etcd-server-tls", - MountPath: "/var/etcd/ssl/client/server", - }, corev1.VolumeMount{ - Name: "client-url-etcd-client-tls", - MountPath: "/var/etcd/ssl/client/client", - }) - } - - if peerUrlTLS != nil { - vms = append(vms, corev1.VolumeMount{ - Name: "peer-url-ca-etcd", - MountPath: "/var/etcd/ssl/peer/ca", - }, corev1.VolumeMount{ - Name: "peer-url-etcd-server-tls", - MountPath: "/var/etcd/ssl/peer/server", - }) - } - - return vms -} - -func getBackupRestoreVolumeMounts(c *component) []corev1.VolumeMount { - vms := []corev1.VolumeMount{ - { - Name: c.values.VolumeClaimTemplateName, - MountPath: "/var/etcd/data", - }, - { - Name: "etcd-config-file", - MountPath: "/var/etcd/config/", - }, - } - - vms = append(vms, getSecretVolumeMounts(c.values.ClientUrlTLS, c.values.PeerUrlTLS)...) - - if c.values.BackupStore == nil { - return vms - } - - provider, err := utils.StorageProviderFromInfraProvider(c.values.BackupStore.Provider) - if err != nil { - return vms - } - - switch provider { - case utils.Local: - if c.values.BackupStore.Container != nil { - if c.featureGates["UseEtcdWrapper"] { - vms = append(vms, corev1.VolumeMount{ - Name: "host-storage", - MountPath: "/home/nonroot/" + pointer.StringDeref(c.values.BackupStore.Container, ""), - }) - } else { - vms = append(vms, corev1.VolumeMount{ - Name: "host-storage", - MountPath: pointer.StringDeref(c.values.BackupStore.Container, ""), - }) - } - } - case utils.GCS: - vms = append(vms, corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/.gcp/", - }) - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - vms = append(vms, corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/etcd-backup/", - }) - } - - return vms -} - -func getStorageReq(val Values) corev1.ResourceRequirements { - storageCapacity := defaultStorageCapacity - if val.StorageCapacity != nil { - storageCapacity = *val.StorageCapacity - } - - return corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: storageCapacity, - }, - } -} - -func getBackupPorts(val Values) []corev1.ContainerPort { - return []corev1.ContainerPort{ - { - Name: "server", - Protocol: "TCP", - ContainerPort: pointer.Int32Deref(val.BackupPort, defaultBackupPort), - }, - } -} - -func getBackupResources(val Values) corev1.ResourceRequirements { - if val.BackupResourceRequirements != nil { - return *val.BackupResourceRequirements - } - return defaultResourceRequirements -} - -func getVolumes(ctx context.Context, cl client.Client, logger logr.Logger, val Values) ([]corev1.Volume, error) { - vs := []corev1.Volume{ - { - Name: "etcd-config-file", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: val.ConfigMapName, - }, - Items: []corev1.KeyToPath{ - { - Key: "etcd.conf.yaml", - Path: "etcd.conf.yaml", - }, - }, - DefaultMode: pointer.Int32(0644), - }, - }, - }, - } - - if val.ClientUrlTLS != nil { - vs = append(vs, corev1.Volume{ - Name: "client-url-ca-etcd", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: val.ClientUrlTLS.TLSCASecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "client-url-etcd-server-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: val.ClientUrlTLS.ServerTLSSecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "client-url-etcd-client-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: val.ClientUrlTLS.ClientTLSSecretRef.Name, - }, - }, - }) - } - - if val.PeerUrlTLS != nil { - vs = append(vs, corev1.Volume{ - Name: "peer-url-ca-etcd", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: val.PeerUrlTLS.TLSCASecretRef.Name, - }, - }, - }, - corev1.Volume{ - Name: "peer-url-etcd-server-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: val.PeerUrlTLS.ServerTLSSecretRef.Name, - }, - }, - }) - } - - if val.BackupStore == nil { - return vs, nil - } - - storeValues := val.BackupStore - provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) - if err != nil { - return vs, nil - } - - switch provider { - case "Local": - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, val.Namespace) - if err != nil { - return nil, err - } - - hpt := corev1.HostPathDirectory - vs = append(vs, corev1.Volume{ - Name: "host-storage", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: hostPath + "/" + pointer.StringDeref(storeValues.Container, ""), - Type: &hpt, - }, - }, - }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: - if storeValues.SecretRef == nil { - return nil, fmt.Errorf("no secretRef configured for backup store") - } - - vs = append(vs, corev1.Volume{ - Name: "etcd-backup", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, - }, - }, - }) - } - - return vs, nil -} - -func getReadinessHandler(val Values) corev1.ProbeHandler { - if val.Replicas > 1 { - // TODO(timuthy): Special handling for multi-node etcd can be removed as soon as - // etcd-backup-restore supports `/healthz` for etcd followers, see https://github.com/gardener/etcd-backup-restore/pull/491. - return getReadinessHandlerForMultiNode(val) - } - return getReadinessHandlerForSingleNode(val) -} - -func getReadinessHandlerForSingleNode(val Values) corev1.ProbeHandler { - scheme := corev1.URISchemeHTTPS - if val.BackupTLS == nil { - scheme = corev1.URISchemeHTTP - } - - return corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(int(pointer.Int32Deref(val.BackupPort, defaultBackupPort))), - Scheme: scheme, - }, - } -} - -func getReadinessHandlerForMultiNode(val Values) corev1.ProbeHandler { - if val.UseEtcdWrapper { - //TODO @aaronfern: remove this feature gate when UseEtcdWrapper becomes GA - scheme := corev1.URISchemeHTTPS - if val.BackupTLS == nil { - scheme = corev1.URISchemeHTTP - } - - return corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromInt(int(pointer.Int32Deref(val.WrapperPort, defaultWrapperPort))), - Scheme: scheme, - }, - } - } - - return corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: val.ReadinessProbeCommand, - }, - } -} - -func getOwnerReferenceNameWithUID(ref metav1.OwnerReference) string { - return fmt.Sprintf("%s:%s", ref.Name, ref.UID) -} diff --git a/pkg/component/etcd/statefulset/statefulset_suite_test.go b/pkg/component/etcd/statefulset/statefulset_suite_test.go deleted file mode 100644 index c7ab167a5..000000000 --- a/pkg/component/etcd/statefulset/statefulset_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package statefulset_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestStatefulSet(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Statefulset Component Suite") -} diff --git a/pkg/component/etcd/statefulset/statefulset_test.go b/pkg/component/etcd/statefulset/statefulset_test.go deleted file mode 100644 index 9bce08ff5..000000000 --- a/pkg/component/etcd/statefulset/statefulset_test.go +++ /dev/null @@ -1,1093 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package statefulset_test - -import ( - "context" - "fmt" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/etcd-druid/pkg/common" - . "github.com/gardener/etcd-druid/pkg/component/etcd/statefulset" - druidutils "github.com/gardener/etcd-druid/pkg/utils" - testutils "github.com/gardener/etcd-druid/test/utils" - - "github.com/gardener/gardener/pkg/component" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - . "github.com/gardener/gardener/pkg/utils/test/matchers" - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - gomegatypes "github.com/onsi/gomega/types" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/component-base/featuregate" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var ( - backupRestore = "backup-restore" - deltaSnapshotPeriod = metav1.Duration{ - Duration: 300 * time.Second, - } - garbageCollectionPeriod = metav1.Duration{ - Duration: 43200 * time.Second, - } - checkSumAnnotations = map[string]string{ - "checksum/etcd-configmap": "abc123", - } - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - wrapperPort int32 = 9095 - uid = "a9b8c7d6e5f4" - imageEtcd = "europe-docker.pkg.dev/gardener-project/releases/gardener/etcd-wrapper:v0.1.0" - imageBR = "europe-docker.pkg.dev/gardener-project/releases/gardener/etcdbrctl:v0.25.0" - imageInitContainer = "europe-docker.pkg.dev/gardener-project/releases/3rd/alpine:3.18.4" - snapshotSchedule = "0 */24 * * *" - defragSchedule = "0 */24 * * *" - container = "default.bkp" - storageCapacity = resource.MustParse("5Gi") - storageClass = "gardener.fast" - priorityClassName = "class_priority" - deltaSnapShotMemLimit = resource.MustParse("100Mi") - autoCompactionMode = druidv1alpha1.Periodic - autoCompactionRetention = "2m" - quota = resource.MustParse("8Gi") - prefix = "/tmp" - volumeClaimTemplateName = "etcd-test" - garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) - metricsBasic = druidv1alpha1.Basic - etcdSnapshotTimeout = metav1.Duration{ - Duration: 10 * time.Minute, - } - etcdDefragTimeout = metav1.Duration{ - Duration: 10 * time.Minute, - } - etcdLeaderElectionConnectionTimeout = metav1.Duration{ - Duration: 5 * time.Second, - } - heartbeatDuration = metav1.Duration{ - Duration: 10 * time.Second, - } - backupRestoreResources = corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("2Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("23m"), - "memory": testutils.ParseQuantity("128Mi"), - }, - } - etcdResources = corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("2500m"), - "memory": testutils.ParseQuantity("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("1000Mi"), - }, - } -) - -var _ = Describe("Statefulset", func() { - var ( - ctx context.Context - cl client.Client - err error - - etcd *druidv1alpha1.Etcd - namespace string - name string - - replicas *int32 - sts *appsv1.StatefulSet - - values *Values - stsDeployer component.Deployer - - storageProvider *string - ) - - JustBeforeEach(func() { - etcd = getEtcd(name, namespace, true, *replicas, storageProvider) - values, err = GenerateValues( - etcd, - pointer.Int32(clientPort), - pointer.Int32(serverPort), - pointer.Int32(backupPort), - imageEtcd, - imageBR, - imageInitContainer, - checkSumAnnotations, - false, - true, - ) - Expect(err).ToNot(HaveOccurred()) - fg := map[featuregate.Feature]bool{ - "UseEtcdWrapper": true, - } - stsDeployer = New(cl, logr.Discard(), *values, fg) - - sts = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: values.Name, - Namespace: values.Namespace, - }, - } - }) - - BeforeEach(func() { - ctx = context.Background() - cl = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - - name = "statefulset" - namespace = "default" - quota = resource.MustParse("8Gi") - - if replicas == nil { - replicas = pointer.Int32(1) - } - }) - - AfterEach(func() { - storageProvider = nil - }) - - Describe("#Deploy", func() { - Context("when statefulset does not exist", func() { - Context("bootstrap of a single replica statefulset", func() { - It("should create the statefulset successfully", func() { - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - sts := &appsv1.StatefulSet{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed()) - checkStatefulset(sts, *values) - }) - }) - - Context("bootstrap of a multi replica statefulset", func() { - BeforeEach(func() { - replicas = pointer.Int32(3) - }) - It("should create the statefulset successfully", func() { - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - sts := &appsv1.StatefulSet{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed()) - checkStatefulset(sts, *values) - // ensure that scale-up annotation "gardener.cloud/scaled-to-multi-node" is not there - Expect(metav1.HasAnnotation(sts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeFalse()) - }) - }) - - Context("DeltaSnapshotRetentionPeriod field is set in Etcd CRD", func() { - It("should include --delta-snapshot-retention-period flag in etcd-backup-restore container command", func() { - etcd.Spec.Backup.DeltaSnapshotRetentionPeriod = &metav1.Duration{Duration: time.Hour * 24} - fg := map[featuregate.Feature]bool{ - "UseEtcdWrapper": true, - } - stsDeployer = New(cl, logr.Discard(), *values, fg) - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - }) - }) - }) - - Context("when statefulset exists", func() { - It("should update the statefulset successfully", func() { - // The generation is usually increased by the Kube-Apiserver but as we use a fake client here, we need to manually do it. - sts.Generation = 1 - Expect(cl.Create(ctx, sts)).To(Succeed()) - - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - }) - - Context("when multi-node cluster is configured", func() { - var ( - annotations = make(map[string]string) - ) - BeforeEach(func() { - replicas = pointer.Int32(3) - annotations = map[string]string{ - ScaleToMultiNodeAnnotationKey: "", - } - }) - - It("should add scale-up annotation to statefulset", func() { - sts.Generation = 1 - sts.Spec.Replicas = pointer.Int32(1) - sts.Status.AvailableReplicas = 1 - Expect(cl.Create(ctx, sts)).To(Succeed()) - Expect(metav1.HasAnnotation(sts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeFalse()) - - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - // ensure that scale-up annotation "gardener.cloud/scaled-to-multi-node" should be added to statefulset. - Expect(metav1.HasAnnotation(updatedSts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeTrue()) - }) - It("shouldn't remove the scale-up annotation from statefulset if scale-up is not completed yet", func() { - sts = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: values.Name, - Namespace: values.Namespace, - // add the scale-up annotation to statefulset - Annotations: annotations, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: pointer.Int32(3), - }, - Status: appsv1.StatefulSetStatus{ - // assuming scale-up isn't completed yet, hence - // set the UpdatedReplicas to 2 - UpdatedReplicas: 2, - AvailableReplicas: 3, - Replicas: 3, - }, - } - - Expect(cl.Create(ctx, sts)).To(Succeed()) - Expect(metav1.HasAnnotation(sts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeTrue()) - - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - // ensure that scale-up annotation "gardener.cloud/scaled-to-multi-node" - // shouldn't be removed from statefulset until scale-up succeeds. - Expect(metav1.HasAnnotation(updatedSts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeTrue()) - }) - It("shouldn't remove the scale-up annotation from statefulset if scale-up is not successful yet", func() { - sts = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: values.Name, - Namespace: values.Namespace, - // add the scale-up annotation to statefulset - Annotations: annotations, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: pointer.Int32(3), - }, - Status: appsv1.StatefulSetStatus{ - // assuming scale-up isn't successful yet, hence - // set the UpdatedReplicas to 3 as pod spec has been updated for all members, - // but set AvailableReplicas is 2 as one pod member unable to come-up. - UpdatedReplicas: 3, - AvailableReplicas: 2, - Replicas: 3, - }, - } - - Expect(cl.Create(ctx, sts)).To(Succeed()) - Expect(metav1.HasAnnotation(sts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeTrue()) - - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - // ensure that scale-up annotation "gardener.cloud/scaled-to-multi-node" - // shouldn't be removed from statefulset until scale-up succeeds. - Expect(metav1.HasAnnotation(updatedSts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeTrue()) - }) - It("should remove the scale-up annotation from statefulset after scale-up succeeds", func() { - sts = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: values.Name, - Namespace: values.Namespace, - // add the scale-up annotation to statefulset - Annotations: annotations, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: pointer.Int32(3), - }, - Status: appsv1.StatefulSetStatus{ - // scale-up is successful, hence - // set the Replicas, UpdatedReplicas and AvailableReplicas to `3`. - UpdatedReplicas: 3, - AvailableReplicas: 3, - Replicas: 3, - }, - } - - Expect(cl.Create(ctx, sts)).To(Succeed()) - Expect(metav1.HasAnnotation(sts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeTrue()) - - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - // After scale-up succeeds ensure that scale-up annotation "gardener.cloud/scaled-to-multi-node" - // should be removed from statefulset. - Expect(metav1.HasAnnotation(updatedSts.ObjectMeta, ScaleToMultiNodeAnnotationKey)).To(BeFalse()) - }) - It("should re-create statefulset because serviceName is changed", func() { - sts.Generation = 2 - sts.Spec.ServiceName = "foo" - sts.Spec.Replicas = pointer.Int32(3) - Expect(cl.Create(ctx, sts)).To(Succeed()) - - values.Replicas = 3 - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - Expect(updatedSts.Spec.ServiceName).To(Equal(values.PeerServiceName)) - }) - - It("should re-create statefulset because podManagementPolicy is changed", func() { - sts.Generation = 2 - sts.Spec.PodManagementPolicy = appsv1.OrderedReadyPodManagement - sts.Spec.ServiceName = values.PeerServiceName - sts.Spec.Replicas = pointer.Int32(3) - Expect(cl.Create(ctx, sts)).To(Succeed()) - - values.Replicas = 3 - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - updatedSts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), updatedSts)).To(Succeed()) - checkStatefulset(updatedSts, *values) - Expect(updatedSts.Spec.PodManagementPolicy).To(Equal(appsv1.ParallelPodManagement)) - }) - }) - - Context("DeltaSnapshotRetentionPeriod field is updated in Etcd CRD", func() { - It("should update --delta-snapshot-retention-period flag in etcd-backup-restore container command", func() { - etcd.Spec.Backup.DeltaSnapshotRetentionPeriod = &metav1.Duration{Duration: time.Hour * 48} - values, err = GenerateValues( - etcd, - pointer.Int32(clientPort), - pointer.Int32(serverPort), - pointer.Int32(backupPort), - imageEtcd, - imageBR, - imageInitContainer, - checkSumAnnotations, false, true) - Expect(err).ToNot(HaveOccurred()) - fg := map[featuregate.Feature]bool{ - "UseEtcdWrapper": true, - } - stsDeployer = New(cl, logr.Discard(), *values, fg) - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - - sts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed()) - checkStatefulset(sts, *values) - }) - }) - }) - - Context("with backup", func() { - for _, p := range []string{ - druidutils.ABS, - druidutils.GCS, - druidutils.S3, - druidutils.Swift, - druidutils.OSS, - druidutils.OCS, - } { - provider := p - Context(fmt.Sprintf("with provider %s", provider), func() { - BeforeEach(func() { - storageProvider = &provider - }) - - It("should configure the correct provider values", func() { - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - sts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed()) - - checkBackup(etcd, sts) - }) - }) - } - - Context("with provider Local", func() { - var ( - backupSecretData map[string][]byte - hostPath string - ) - - BeforeEach(func() { - storageProvider = pointer.String(druidutils.Local) - }) - - JustBeforeEach(func() { - Expect(cl.Create(ctx, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: etcdBackupSecretName, - Namespace: namespace, - }, - Data: backupSecretData, - })).To(Succeed()) - }) - - Context("when backup secret defines a hostPath", func() { - BeforeEach(func() { - hostPath = "/data" - backupSecretData = map[string][]byte{ - druidutils.EtcdBackupSecretHostPath: []byte(hostPath), - } - }) - - It("should configure the correct provider values", func() { - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - sts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed()) - - checkLocalProviderValues(etcd, sts, hostPath) - }) - }) - - Context("when backup secret doesn't define a hostPath", func() { - BeforeEach(func() { - backupSecretData = map[string][]byte{ - "foo": []byte("bar"), - } - }) - - It("should configure the correct provider values", func() { - Expect(stsDeployer.Deploy(ctx)).To(Succeed()) - sts := &appsv1.StatefulSet{} - Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed()) - - checkLocalProviderValues(etcd, sts, druidutils.LocalProviderDefaultMountPath) - }) - }) - }) - }) - }) - - Describe("#Destroy", func() { - Context("when statefulset does not exist", func() { - It("should destroy successfully", func() { - Expect(stsDeployer.Destroy(ctx)).To(Succeed()) - Expect(cl.Get(ctx, client.ObjectKeyFromObject(sts), &appsv1.StatefulSet{})).To(BeNotFoundError()) - }) - }) - - Context("when statefulset exists", func() { - It("should destroy successfully", func() { - Expect(cl.Create(ctx, sts)).To(Succeed()) - - Expect(stsDeployer.Destroy(ctx)).To(Succeed()) - - Expect(cl.Get(ctx, kutil.Key(namespace, sts.Name), &appsv1.StatefulSet{})).To(BeNotFoundError()) - }) - }) - }) -}) - -func checkBackup(etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) { - // Check secret volume mount - Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(corev1.Volume{ - Name: "etcd-backup", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Backup.Store.SecretRef.Name, - }, - }, - })) - - backupRestoreContainer := sts.Spec.Template.Spec.Containers[1] - Expect(backupRestoreContainer.Name).To(Equal(backupRestore)) - - mountPath := "/var/etcd-backup/" - if *etcd.Spec.Backup.Store.Provider == druidutils.GCS { - mountPath = "/var/.gcp/" - } - - // Check volume mount - Expect(backupRestoreContainer.VolumeMounts).To(ContainElement(corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: mountPath, - })) - - // Check command - Expect(backupRestoreContainer.Args).To(ContainElements( - "--storage-provider="+string(*etcd.Spec.Backup.Store.Provider), - "--store-prefix="+prefix, - )) - - var ( - envVarName string - envVarValue = "/var/etcd-backup" - ) - - switch *etcd.Spec.Backup.Store.Provider { - case druidutils.S3: - envVarName = common.EnvAWSApplicationCredentials - - case druidutils.ABS: - envVarName = common.EnvAzureApplicationCredentials - - case druidutils.GCS: - envVarName = common.EnvGoogleApplicationCredentials - envVarValue = "/var/.gcp/serviceaccount.json" - - case druidutils.Swift: - envVarName = common.EnvOpenstackApplicationCredentials - - case druidutils.OSS: - envVarName = common.EnvAlicloudApplicationCredentials - - case druidutils.OCS: - envVarName = common.EnvOpenshiftApplicationCredentials - } - - // Check env var - Expect(backupRestoreContainer.Env).To(ContainElement(corev1.EnvVar{ - Name: envVarName, - Value: envVarValue, - })) -} - -func checkStatefulset(sts *appsv1.StatefulSet, values Values) { - checkStsOwnerRefs(sts.ObjectMeta.OwnerReferences, values) - Expect(*sts).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(values.Name), - "Namespace": Equal(values.Namespace), - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "checksum/etcd-configmap": Equal("abc123"), - "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", values.Namespace, values.Name)), - "gardener.cloud/owner-type": Equal("etcd"), - "app": Equal("etcd-statefulset"), - "role": Equal("test"), - "instance": Equal(values.Name), - }), - "Labels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(values.Name), - }), - }), - - "Spec": MatchFields(IgnoreExtras, Fields{ - "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ - "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), - }), - "Replicas": PointTo(Equal(values.Replicas)), - "Selector": PointTo(MatchFields(IgnoreExtras, Fields{ - "MatchLabels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(values.Name), - }), - })), - "Template": MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "app": Equal("etcd-statefulset"), - "role": Equal("test"), - "instance": Equal(values.Name), - }), - "Labels": MatchAllKeys(Keys{ - "foo": Equal("bar"), - "name": Equal("etcd"), - "instance": Equal(values.Name), - }), - }), - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "HostAliases": MatchAllElements(hostAliasIterator, Elements{ - "127.0.0.1": MatchFields(IgnoreExtras, Fields{ - "IP": Equal("127.0.0.1"), - "Hostnames": MatchAllElements(cmdIterator, Elements{ - fmt.Sprintf("%s-local", values.Name): Equal(fmt.Sprintf("%s-local", values.Name)), - }), - }), - }), - "Containers": MatchAllElements(containerIterator, Elements{ - common.Etcd: MatchFields(IgnoreExtras, Fields{ - "Ports": ConsistOf([]corev1.ContainerPort{ - { - Name: "server", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: *values.ServerPort, - }, - { - Name: "client", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: *values.ClientPort, - }, - }), - "Args": MatchAllElements(cmdIterator, Elements{ - "start-etcd": Equal("start-etcd"), - "--backup-restore-tls-enabled=true": Equal("--backup-restore-tls-enabled=true"), - "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt": Equal("--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt"), - "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key": Equal("--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key"), - "--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/ca.crt": Equal("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/ca.crt"), - fmt.Sprintf("--backup-restore-host-port=%s-local:%d", values.Name, backupPort): Equal(fmt.Sprintf("--backup-restore-host-port=%s-local:%d", values.Name, backupPort)), - fmt.Sprintf("--etcd-server-name=%s-local", values.Name): Equal(fmt.Sprintf("--etcd-server-name=%s-local", values.Name)), - }), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "Image": Equal(values.EtcdImage), - "ReadinessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ - "ProbeHandler": getReadinessHandler(values), - "InitialDelaySeconds": Equal(int32(15)), - "PeriodSeconds": Equal(int32(5)), - "FailureThreshold": Equal(int32(5)), - })), - "Resources": Equal(etcdResources), - "VolumeMounts": MatchAllElements(volumeMountIterator, Elements{ - values.VolumeClaimTemplateName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(values.VolumeClaimTemplateName), - "MountPath": Equal("/var/etcd/data/"), - }), - "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-ca-etcd"), - "MountPath": Equal("/var/etcd/ssl/client/ca"), - }), - "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-server-tls"), - "MountPath": Equal("/var/etcd/ssl/client/server"), - }), - "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-client-tls"), - "MountPath": Equal("/var/etcd/ssl/client/client"), - }), - "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-ca-etcd"), - "MountPath": Equal("/var/etcd/ssl/peer/ca"), - }), - "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-etcd-server-tls"), - "MountPath": Equal("/var/etcd/ssl/peer/server"), - }), - }), - }), - - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchAllElements(cmdIterator, expectedBackupArgs(&values)), - "Ports": ConsistOf([]corev1.ContainerPort{ - { - Name: "server", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: *values.BackupPort, - }, - }), - "Image": Equal(values.BackupImage), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "VolumeMounts": MatchElements(volumeMountIterator, IgnoreExtras, Elements{ - values.VolumeClaimTemplateName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(values.VolumeClaimTemplateName), - "MountPath": Equal("/var/etcd/data"), - }), - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "MountPath": Equal("/var/etcd/config/"), - }), - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "MountPath": Equal("/var/etcd-backup/"), - }), - }), - "Env": MatchElements(envIterator, IgnoreExtras, Elements{ - common.EnvStorageContainer: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*values.BackupStore.Container), - }), - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvAzureApplicationCredentials), - "Value": Equal("/var/etcd-backup"), - }), - }), - "Resources": Equal(backupRestoreResources), - "SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{ - "Capabilities": PointTo(MatchFields(IgnoreExtras, Fields{ - "Add": ConsistOf([]corev1.Capability{ - "SYS_PTRACE", - }), - })), - })), - }), - }), - "ShareProcessNamespace": Equal(pointer.Bool(true)), - "Volumes": MatchAllElements(volumeIterator, Elements{ - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ - "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(fmt.Sprintf("etcd-bootstrap-%s", string(values.OwnerReference.UID[:6]))), - }), - "DefaultMode": PointTo(Equal(int32(0644))), - "Items": MatchAllElements(keyIterator, Elements{ - "etcd.conf.yaml": MatchFields(IgnoreExtras, Fields{ - "Key": Equal("etcd.conf.yaml"), - "Path": Equal("etcd.conf.yaml"), - }), - }), - })), - }), - }), - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(values.BackupStore.SecretRef.Name), - })), - }), - }), - "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-server-tls"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(values.ClientUrlTLS.ServerTLSSecretRef.Name), - })), - }), - }), - "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-client-tls"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(values.ClientUrlTLS.ClientTLSSecretRef.Name), - })), - }), - }), - "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-ca-etcd"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(values.ClientUrlTLS.TLSCASecretRef.Name), - })), - }), - }), - "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-etcd-server-tls"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(values.PeerUrlTLS.ServerTLSSecretRef.Name), - })), - }), - }), - "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-ca-etcd"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(values.PeerUrlTLS.TLSCASecretRef.Name), - })), - }), - }), - }), - }), - }), - "VolumeClaimTemplates": MatchAllElements(pvcIterator, Elements{ - values.VolumeClaimTemplateName: MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(values.VolumeClaimTemplateName), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "StorageClassName": PointTo(Equal(*values.StorageClass)), - "AccessModes": MatchAllElements(accessModeIterator, Elements{ - "ReadWriteOnce": Equal(corev1.ReadWriteOnce), - }), - "Resources": MatchFields(IgnoreExtras, Fields{ - "Requests": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceStorage: Equal(*values.StorageCapacity), - }), - }), - }), - }), - }), - }), - })) -} - -func checkStsOwnerRefs(ors []metav1.OwnerReference, values Values) { - Expect(ors).To(Equal([]metav1.OwnerReference{values.OwnerReference})) -} - -func getEtcd(name, namespace string, tlsEnabled bool, replicas int32, storageProvider *string) *druidv1alpha1.Etcd { - instance := &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: types.UID(uid), - }, - Spec: druidv1alpha1.EtcdSpec{ - Annotations: map[string]string{ - "app": "etcd-statefulset", - "role": "test", - "instance": name, - }, - Labels: map[string]string{ - "foo": "bar", - }, - Replicas: replicas, - StorageCapacity: &storageCapacity, - StorageClass: &storageClass, - PriorityClassName: &priorityClassName, - VolumeClaimTemplate: &volumeClaimTemplateName, - Backup: druidv1alpha1.BackupSpec{ - Image: &imageBR, - Port: pointer.Int32(backupPort), - Store: getEtcdBackup(storageProvider), - FullSnapshotSchedule: &snapshotSchedule, - GarbageCollectionPolicy: &garbageCollectionPolicy, - GarbageCollectionPeriod: &garbageCollectionPeriod, - DeltaSnapshotPeriod: &deltaSnapshotPeriod, - DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, - EtcdSnapshotTimeout: &etcdSnapshotTimeout, - LeaderElection: &druidv1alpha1.LeaderElectionSpec{ - EtcdConnectionTimeout: &etcdLeaderElectionConnectionTimeout, - }, - - Resources: &backupRestoreResources, - }, - Etcd: druidv1alpha1.EtcdConfig{ - Quota: "a, - Metrics: &metricsBasic, - Image: &imageEtcd, - DefragmentationSchedule: &defragSchedule, - EtcdDefragTimeout: &etcdDefragTimeout, - HeartbeatDuration: &heartbeatDuration, - Resources: &etcdResources, - ClientPort: pointer.Int32(clientPort), - ServerPort: pointer.Int32(serverPort), - }, - Common: druidv1alpha1.SharedConfig{ - AutoCompactionMode: &autoCompactionMode, - AutoCompactionRetention: &autoCompactionRetention, - }, - }, - Status: druidv1alpha1.EtcdStatus{ - Replicas: 0, - }, - } - - if tlsEnabled { - clientTlsConfig := &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "client-url-ca-etcd", - }, - DataKey: pointer.String("ca.crt"), - }, - ClientTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-client-tls", - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-server-tls", - }, - } - - peerTlsConfig := &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "peer-url-ca-etcd", - }, - DataKey: pointer.String("ca.crt"), - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "peer-url-etcd-server-tls", - }, - } - - instance.Spec.Etcd.ClientUrlTLS = clientTlsConfig - instance.Spec.Etcd.PeerUrlTLS = peerTlsConfig - instance.Spec.Backup.TLS = clientTlsConfig - } - return instance -} - -const etcdBackupSecretName = "etcd-backup" - -func getEtcdBackup(provider *string) *druidv1alpha1.StoreSpec { - storageProvider := pointer.StringDeref(provider, druidutils.ABS) - - return &druidv1alpha1.StoreSpec{ - Container: &container, - Prefix: prefix, - Provider: (*druidv1alpha1.StorageProvider)(&storageProvider), - SecretRef: &corev1.SecretReference{ - Name: etcdBackupSecretName, - }, - } -} - -func volumeMountIterator(element interface{}) string { - return (element.(corev1.VolumeMount)).Name -} - -func volumeIterator(element interface{}) string { - return (element.(corev1.Volume)).Name -} - -func keyIterator(element interface{}) string { - return (element.(corev1.KeyToPath)).Key -} - -func envIterator(element interface{}) string { - return (element.(corev1.EnvVar)).Name -} - -func containerIterator(element interface{}) string { - return (element.(corev1.Container)).Name -} - -func hostAliasIterator(element interface{}) string { - return (element.(corev1.HostAlias)).IP -} - -func pvcIterator(element interface{}) string { - return (element.(corev1.PersistentVolumeClaim)).Name -} - -func accessModeIterator(element interface{}) string { - return string(element.(corev1.PersistentVolumeAccessMode)) -} - -func cmdIterator(element interface{}) string { - return element.(string) -} - -func getReadinessHandler(val Values) gomegatypes.GomegaMatcher { - if val.Replicas > 1 { - return getReadinessHandlerForMultiNode() - } - return getReadinessHandlerForSingleNode() -} - -func getReadinessHandlerForSingleNode() gomegatypes.GomegaMatcher { - return MatchFields(IgnoreExtras, Fields{ - "HTTPGet": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal("/healthz"), - "Port": Equal(intstr.FromInt(int(backupPort))), - "Scheme": Equal(corev1.URISchemeHTTPS), - })), - }) -} - -func getReadinessHandlerForMultiNode() gomegatypes.GomegaMatcher { - return MatchFields(IgnoreExtras, Fields{ - "HTTPGet": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal("/readyz"), - "Port": Equal(intstr.FromInt(int(wrapperPort))), - "Scheme": Equal(corev1.URISchemeHTTPS), - })), - }) -} - -func checkLocalProviderValues(etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet, hostPath string) { - hpt := corev1.HostPathDirectory - - // check volumes - ExpectWithOffset(1, sts.Spec.Template.Spec.Volumes).To(ContainElements(corev1.Volume{ - Name: "host-storage", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: hostPath + "/" + container, - Type: &hpt, - }, - }, - })) - - backupRestoreContainer := sts.Spec.Template.Spec.Containers[1] - ExpectWithOffset(1, backupRestoreContainer.Name).To(Equal(backupRestore)) - - // Check command - ExpectWithOffset(1, backupRestoreContainer.Args).To(ContainElements( - "--storage-provider="+string(*etcd.Spec.Backup.Store.Provider), - "--store-prefix="+prefix, - )) - - // check volume mount - ExpectWithOffset(1, backupRestoreContainer.VolumeMounts).To(ContainElement(corev1.VolumeMount{ - Name: "host-storage", - MountPath: "/home/nonroot/" + container, - })) -} - -func expectedBackupArgs(values *Values) Elements { - store, err := druidutils.StorageProviderFromInfraProvider(values.BackupStore.Provider) - Expect(err).NotTo(HaveOccurred()) - elements := Elements{ - "server": Equal("server"), - "--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"), - "--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"), - "--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"), - "--server-cert=/var/etcd/ssl/client/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/client/server/tls.crt"), - "--server-key=/var/etcd/ssl/client/server/tls.key": Equal("--server-key=/var/etcd/ssl/client/server/tls.key"), - "--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"), - "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp"), - "--insecure-transport=false": Equal("--insecure-transport=false"), - "--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"), - "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), - fmt.Sprintf("%s=%s", "--etcd-connection-timeout-leader-election", etcdLeaderElectionConnectionTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-connection-timeout-leader-election", values.LeaderElection.EtcdConnectionTimeout.Duration.String())), - "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), - "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), - "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), - "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), - fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule)), - fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule)), - fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy)), - fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), - fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix)), - fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value())), - fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy)), - fmt.Sprintf("--endpoints=https://%s-local:%d", values.Name, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", values.Name, clientPort)), - fmt.Sprintf("--service-endpoints=https://%s:%d", values.ClientServiceName, clientPort): Equal(fmt.Sprintf("--service-endpoints=https://%s:%d", values.ClientServiceName, clientPort)), - fmt.Sprintf("--embedded-etcd-quota-bytes=%d", values.Quota.Value()): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", values.Quota.Value())), - fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String())), - fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String())), - fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode)), - fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention)), - fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String())), - fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String())), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName)), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName)), - } - - if values.DeltaSnapshotRetentionPeriod != nil { - elements[fmt.Sprintf("--delta-snapshot-retention-period=%s", values.DeltaSnapshotRetentionPeriod.Duration.String())] = Equal(fmt.Sprintf("--delta-snapshot-retention-period=%s", values.DeltaSnapshotRetentionPeriod.Duration.String())) - } - return elements -} diff --git a/pkg/component/etcd/statefulset/values.go b/pkg/component/etcd/statefulset/values.go deleted file mode 100644 index 1ab9f789b..000000000 --- a/pkg/component/etcd/statefulset/values.go +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package statefulset - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/apimachinery/pkg/api/resource" -) - -// Values contains the values necessary for creating ETCD statefulset. -type Values struct { - // Name is the name of the StatefulSet. - Name string - // Namespace is the namespace of StatefulSet. - Namespace string - - // OwnerReference is the OwnerReference for the StatefulSet. - OwnerReference metav1.OwnerReference - - // Replicas is the number of ETCD instance that the ETCD cluster will have. - Replicas int32 - // StatusReplicas is the number of replicas maintained in ETCD status. - StatusReplicas int32 - - // Annotations is the annotation provided in ETCD spec. - Annotations map[string]string - // Labels is the labels of StatefulSet.. - Labels map[string]string - // AdditionalPodLabels represents additional labels to be applied to the StatefulSet pods. - AdditionalPodLabels map[string]string - // BackupImage is the backup restore image. - BackupImage string - // EtcdImage is the etcd custom image. - EtcdImage string - // InitContainerImage is the image used in the init container in the etcd pod - InitContainerImage string - // PriorityClassName is the Priority Class name. - PriorityClassName *string - // ServiceAccountName is the service account name. - ServiceAccountName string - Affinity *corev1.Affinity - TopologySpreadConstraints []corev1.TopologySpreadConstraint - - EtcdResourceRequirements *corev1.ResourceRequirements - BackupResourceRequirements *corev1.ResourceRequirements - - EtcdCommandArgs []string - ReadinessProbeCommand []string - EtcdBackupRestoreCommandArgs []string - - EnableClientTLS string - EnablePeerTLS string - - FailBelowRevision string - VolumeClaimTemplateName string - - FullSnapLeaseName string - DeltaSnapLeaseName string - - StorageCapacity *resource.Quantity - StorageClass *string - - DefragmentationSchedule *string - FullSnapshotSchedule *string - - EtcdSnapshotTimeout *metav1.Duration - EtcdDefragTimeout *metav1.Duration - - DeltaSnapshotMemoryLimit *resource.Quantity - - GarbageCollectionPolicy *druidv1alpha1.GarbageCollectionPolicy - MaxBackupsLimitBasedGC *int32 - GarbageCollectionPeriod *metav1.Duration - - LeaderElection *druidv1alpha1.LeaderElectionSpec - BackupStore *druidv1alpha1.StoreSpec - - EnableProfiling *bool - - DeltaSnapshotPeriod *metav1.Duration - DeltaSnapshotRetentionPeriod *metav1.Duration - - SnapshotCompression *druidv1alpha1.CompressionSpec - HeartbeatDuration *metav1.Duration - - // MetricsLevel defines the level of detail for exported metrics of etcd, specify 'extensive' to include histogram metrics. - MetricsLevel *druidv1alpha1.MetricsLevel - // Quota defines the etcd DB quota. - Quota *resource.Quantity - - // ClientUrlTLS holds the TLS configuration details for client communication. - ClientUrlTLS *druidv1alpha1.TLSConfig - // PeerUrlTLS hold the TLS configuration details for peer communication. - PeerUrlTLS *druidv1alpha1.TLSConfig - // BackupTLS hold the TLS configuration for communication with Backup server. - BackupTLS *druidv1alpha1.TLSConfig - - //ClientServiceName is name of the etcd client service. - ClientServiceName string - // ClientPort holds the client port. - ClientPort *int32 - //PeerServiceName is name of the etcd peer service. - PeerServiceName string - // ServerPort is the peer port. - ServerPort *int32 - // BackupPort is the backup-restore side-car port. - BackupPort *int32 - // WrapperPort is the port where etcd-wrapper registers and exposes it's ready endpoint - WrapperPort *int32 - - // AutoCompactionMode defines the auto-compaction-mode: 'periodic' or 'revision'. - AutoCompactionMode *druidv1alpha1.CompactionMode - //AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. - AutoCompactionRetention *string - // ConfigMapName is the name of the configmap that holds the ETCD config. - ConfigMapName string - PeerTLSChangedToEnabled bool - - // UseEtcdWrapper enables the use of etcd-wrapper image and a compatible version of etcd-backup-restore - UseEtcdWrapper bool -} diff --git a/pkg/component/etcd/statefulset/values_helper.go b/pkg/component/etcd/statefulset/values_helper.go deleted file mode 100644 index d7bf0ec29..000000000 --- a/pkg/component/etcd/statefulset/values_helper.go +++ /dev/null @@ -1,349 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package statefulset - -import ( - "fmt" - "strings" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/utils" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/utils/pointer" -) - -const ( - defaultBackupPort int32 = 8080 - defaultServerPort int32 = 2380 - defaultClientPort int32 = 2379 - defaultWrapperPort int32 = 9095 - defaultMaxBackupsLimitBasedGC int32 = 7 - defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi - defaultSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi - defaultHeartbeatDuration = "10s" - defaultGbcPolicy = "LimitBased" - defaultAutoCompactionRetention = "30m" - defaultEtcdSnapshotTimeout = "15m" - defaultEtcdDefragTimeout = "15m" - defaultAutoCompactionMode = "periodic" - defaultEtcdConnectionTimeout = "5m" -) - -var defaultStorageCapacity = resource.MustParse("16Gi") - -// GenerateValues generates `statefulset.Values` for the statefulset component with the given parameters. -func GenerateValues( - etcd *druidv1alpha1.Etcd, - clientPort, serverPort, backupPort *int32, - etcdImage, backupImage, initContainerImage string, - checksumAnnotations map[string]string, - peerTLSChangedToEnabled, useEtcdWrapper bool) (*Values, error) { - - volumeClaimTemplateName := etcd.Name - if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { - volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate - } - - values := Values{ - Name: etcd.Name, - Namespace: etcd.Namespace, - OwnerReference: etcd.GetAsOwnerReference(), - Replicas: etcd.Spec.Replicas, - StatusReplicas: etcd.Status.Replicas, - Annotations: utils.MergeStringMaps(checksumAnnotations, etcd.Spec.Annotations), - Labels: etcd.GetDefaultLabels(), - AdditionalPodLabels: etcd.Spec.Labels, - EtcdImage: etcdImage, - BackupImage: backupImage, - InitContainerImage: initContainerImage, - PriorityClassName: etcd.Spec.PriorityClassName, - ServiceAccountName: etcd.GetServiceAccountName(), - Affinity: etcd.Spec.SchedulingConstraints.Affinity, - TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, - - EtcdResourceRequirements: etcd.Spec.Etcd.Resources, - BackupResourceRequirements: etcd.Spec.Backup.Resources, - - VolumeClaimTemplateName: volumeClaimTemplateName, - - FullSnapLeaseName: etcd.GetFullSnapshotLeaseName(), - DeltaSnapLeaseName: etcd.GetDeltaSnapshotLeaseName(), - - StorageCapacity: etcd.Spec.StorageCapacity, - StorageClass: etcd.Spec.StorageClass, - - ClientUrlTLS: etcd.Spec.Etcd.ClientUrlTLS, - PeerUrlTLS: etcd.Spec.Etcd.PeerUrlTLS, - BackupTLS: etcd.Spec.Backup.TLS, - - LeaderElection: etcd.Spec.Backup.LeaderElection, - - BackupStore: etcd.Spec.Backup.Store, - EnableProfiling: etcd.Spec.Backup.EnableProfiling, - - DeltaSnapshotPeriod: etcd.Spec.Backup.DeltaSnapshotPeriod, - DeltaSnapshotRetentionPeriod: etcd.Spec.Backup.DeltaSnapshotRetentionPeriod, - DeltaSnapshotMemoryLimit: etcd.Spec.Backup.DeltaSnapshotMemoryLimit, - - DefragmentationSchedule: etcd.Spec.Etcd.DefragmentationSchedule, - FullSnapshotSchedule: etcd.Spec.Backup.FullSnapshotSchedule, - - EtcdSnapshotTimeout: etcd.Spec.Backup.EtcdSnapshotTimeout, - EtcdDefragTimeout: etcd.Spec.Etcd.EtcdDefragTimeout, - - GarbageCollectionPolicy: etcd.Spec.Backup.GarbageCollectionPolicy, - MaxBackupsLimitBasedGC: etcd.Spec.Backup.MaxBackupsLimitBasedGC, - GarbageCollectionPeriod: etcd.Spec.Backup.GarbageCollectionPeriod, - - SnapshotCompression: etcd.Spec.Backup.SnapshotCompression, - HeartbeatDuration: etcd.Spec.Etcd.HeartbeatDuration, - - MetricsLevel: etcd.Spec.Etcd.Metrics, - Quota: etcd.Spec.Etcd.Quota, - ClientServiceName: etcd.GetClientServiceName(), - ClientPort: clientPort, - PeerServiceName: etcd.GetPeerServiceName(), - ServerPort: serverPort, - BackupPort: backupPort, - WrapperPort: pointer.Int32(defaultWrapperPort), - - AutoCompactionMode: etcd.Spec.Common.AutoCompactionMode, - AutoCompactionRetention: etcd.Spec.Common.AutoCompactionRetention, - ConfigMapName: etcd.GetConfigMapName(), - PeerTLSChangedToEnabled: peerTLSChangedToEnabled, - - UseEtcdWrapper: useEtcdWrapper, - } - - values.EtcdCommandArgs = getEtcdCommandArgs(values) - - // Use linearizability for readiness probe so that pod is only considered ready - // when it has an active connection to the cluster and the cluster maintains a quorum. - values.ReadinessProbeCommand = getProbeCommand(values, linearizable) - - etcdBackupRestoreCommandArgs, err := getBackupRestoreCommandArgs(values) - if err != nil { - return nil, err - } - values.EtcdBackupRestoreCommandArgs = etcdBackupRestoreCommandArgs - - return &values, nil -} - -func getEtcdCommandArgs(val Values) []string { - if !val.UseEtcdWrapper { - // safe to return an empty string array here since etcd-custom-image:v3.4.13-bootstrap-12 (as well as v3.4.26) now uses an entry point that calls bootstrap.sh - return []string{} - } - //TODO @aaronfern: remove this feature gate when UseEtcdWrapper becomes GA - command := []string{"" + "start-etcd"} - command = append(command, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", val.Name)) - command = append(command, fmt.Sprintf("--etcd-server-name=%s-local", val.Name)) - - if val.ClientUrlTLS == nil { - command = append(command, "--backup-restore-tls-enabled=false") - } else { - dataKey := "ca.crt" - if val.ClientUrlTLS.TLSCASecretRef.DataKey != nil { - dataKey = *val.ClientUrlTLS.TLSCASecretRef.DataKey - } - command = append(command, "--backup-restore-tls-enabled=true") - command = append(command, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") - command = append(command, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") - command = append(command, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) - } - - return command -} - -type consistencyLevel string - -const ( - linearizable consistencyLevel = "linearizable" - serializable consistencyLevel = "serializable" -) - -func getProbeCommand(val Values, consistency consistencyLevel) []string { - var etcdCtlCommand strings.Builder - - etcdCtlCommand.WriteString("ETCDCTL_API=3 etcdctl") - - if val.ClientUrlTLS != nil { - dataKey := "ca.crt" - if val.ClientUrlTLS.TLSCASecretRef.DataKey != nil { - dataKey = *val.ClientUrlTLS.TLSCASecretRef.DataKey - } - - etcdCtlCommand.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) - etcdCtlCommand.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") - etcdCtlCommand.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") - etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", val.Name, pointer.Int32Deref(val.ClientPort, defaultClientPort))) - - } else { - etcdCtlCommand.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", val.Name, pointer.Int32Deref(val.ClientPort, defaultClientPort))) - } - - etcdCtlCommand.WriteString(" get foo") - - switch consistency { - case linearizable: - etcdCtlCommand.WriteString(" --consistency=l") - case serializable: - etcdCtlCommand.WriteString(" --consistency=s") - } - - return []string{ - "/bin/sh", - "-ec", - etcdCtlCommand.String(), - } -} - -func getBackupRestoreCommandArgs(val Values) ([]string, error) { - command := []string{"server"} - - if val.BackupStore != nil { - command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--delta-snapshot-lease-name="+val.DeltaSnapLeaseName) - command = append(command, "--full-snapshot-lease-name="+val.FullSnapLeaseName) - } - - if val.DefragmentationSchedule != nil { - command = append(command, "--defragmentation-schedule="+*val.DefragmentationSchedule) - } - - if val.FullSnapshotSchedule != nil { - command = append(command, "--schedule="+*val.FullSnapshotSchedule) - } - - garbageCollectionPolicy := defaultGbcPolicy - if val.GarbageCollectionPolicy != nil { - garbageCollectionPolicy = string(*val.GarbageCollectionPolicy) - } - - command = append(command, "--garbage-collection-policy="+garbageCollectionPolicy) - if garbageCollectionPolicy == "LimitBased" { - command = append(command, "--max-backups="+fmt.Sprint(pointer.Int32Deref(val.MaxBackupsLimitBasedGC, defaultMaxBackupsLimitBasedGC))) - } - - command = append(command, "--data-dir=/var/etcd/data/new.etcd") - command = append(command, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") - - if val.BackupStore != nil { - store, err := utils.StorageProviderFromInfraProvider(val.BackupStore.Provider) - if err != nil { - return nil, err - } - command = append(command, "--storage-provider="+store) - command = append(command, "--store-prefix="+string(val.BackupStore.Prefix)) - } - - var quota = defaultQuota - if val.Quota != nil { - quota = val.Quota.Value() - } - - command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) - - if pointer.BoolDeref(val.EnableProfiling, false) { - command = append(command, "--enable-profiling=true") - } - - if val.ClientUrlTLS != nil { - command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") - command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") - command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+pointer.StringDeref(val.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt")) - command = append(command, "--insecure-transport=false") - command = append(command, "--insecure-skip-tls-verify=false") - command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", val.Name, pointer.Int32Deref(val.ClientPort, defaultClientPort))) - command = append(command, fmt.Sprintf("--service-endpoints=https://%s:%d", val.ClientServiceName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) - } else { - command = append(command, "--insecure-transport=true") - command = append(command, "--insecure-skip-tls-verify=true") - command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", val.Name, pointer.Int32Deref(val.ClientPort, defaultClientPort))) - command = append(command, fmt.Sprintf("--service-endpoints=http://%s:%d", val.ClientServiceName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) - - } - - if val.BackupTLS != nil { - command = append(command, "--server-cert=/var/etcd/ssl/client/server/tls.crt") - command = append(command, "--server-key=/var/etcd/ssl/client/server/tls.key") - } - - command = append(command, "--etcd-connection-timeout="+defaultEtcdConnectionTimeout) - - if val.DeltaSnapshotPeriod != nil { - command = append(command, "--delta-snapshot-period="+val.DeltaSnapshotPeriod.Duration.String()) - } - - if val.DeltaSnapshotRetentionPeriod != nil { - command = append(command, "--delta-snapshot-retention-period="+val.DeltaSnapshotRetentionPeriod.Duration.String()) - } - - var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit - if val.DeltaSnapshotMemoryLimit != nil { - deltaSnapshotMemoryLimit = val.DeltaSnapshotMemoryLimit.Value() - } - - command = append(command, "--delta-snapshot-memory-limit="+fmt.Sprint(deltaSnapshotMemoryLimit)) - - if val.GarbageCollectionPeriod != nil { - command = append(command, "--garbage-collection-period="+val.GarbageCollectionPeriod.Duration.String()) - } - - if val.SnapshotCompression != nil { - if pointer.BoolDeref(val.SnapshotCompression.Enabled, false) { - command = append(command, "--compress-snapshots="+fmt.Sprint(*val.SnapshotCompression.Enabled)) - } - if val.SnapshotCompression.Policy != nil { - command = append(command, "--compression-policy="+string(*val.SnapshotCompression.Policy)) - } - } - - compactionMode := defaultAutoCompactionMode - if val.AutoCompactionMode != nil { - compactionMode = string(*val.AutoCompactionMode) - } - command = append(command, "--auto-compaction-mode="+compactionMode) - - compactionRetention := defaultAutoCompactionRetention - if val.AutoCompactionRetention != nil { - compactionRetention = *val.AutoCompactionRetention - } - command = append(command, "--auto-compaction-retention="+compactionRetention) - - etcdSnapshotTimeout := defaultEtcdSnapshotTimeout - if val.EtcdSnapshotTimeout != nil { - etcdSnapshotTimeout = val.EtcdSnapshotTimeout.Duration.String() - } - command = append(command, "--etcd-snapshot-timeout="+etcdSnapshotTimeout) - - etcdDefragTimeout := defaultEtcdDefragTimeout - if val.EtcdDefragTimeout != nil { - etcdDefragTimeout = val.EtcdDefragTimeout.Duration.String() - } - command = append(command, "--etcd-defrag-timeout="+etcdDefragTimeout) - - command = append(command, "--snapstore-temp-directory=/var/etcd/data/temp") - command = append(command, "--enable-member-lease-renewal=true") - - heartbeatDuration := defaultHeartbeatDuration - if val.HeartbeatDuration != nil { - heartbeatDuration = val.HeartbeatDuration.Duration.String() - } - command = append(command, "--k8s-heartbeat-duration="+heartbeatDuration) - - if val.LeaderElection != nil { - if val.LeaderElection.EtcdConnectionTimeout != nil { - command = append(command, "--etcd-connection-timeout-leader-election="+val.LeaderElection.EtcdConnectionTimeout.Duration.String()) - } - - if val.LeaderElection.ReelectionPeriod != nil { - command = append(command, "--reelection-period="+val.LeaderElection.ReelectionPeriod.Duration.String()) - } - } - - return command, nil -} diff --git a/pkg/features/features.go b/pkg/features/features.go deleted file mode 100644 index 0bd7c799b..000000000 --- a/pkg/features/features.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package features - -import ( - "k8s.io/component-base/featuregate" -) - -const ( - // Every feature should add method here following this template: - // - // // MyFeature enables Foo. - // // owner: @username - // // alpha: v0.X - // MyFeature featuregate.Feature = "MyFeature" - - // UseEtcdWrapper enables the use of etcd-wrapper image and a compatible version - // of etcd-backup-restore, along with component-specific configuration - // changes required for the usage of the etcd-wrapper image. - // owner @unmarshall @aaronfern - // alpha: v0.19 - // beta: v0.22 - UseEtcdWrapper featuregate.Feature = "UseEtcdWrapper" -) - -var defaultFeatures = map[featuregate.Feature]featuregate.FeatureSpec{ - UseEtcdWrapper: {Default: true, PreRelease: featuregate.Beta}, -} - -// GetDefaultFeatures returns the default feature gates known to etcd-druid. -func GetDefaultFeatures() map[featuregate.Feature]featuregate.FeatureSpec { - return defaultFeatures -} diff --git a/pkg/health/condition/builder.go b/pkg/health/condition/builder.go deleted file mode 100644 index f7c48b13b..000000000 --- a/pkg/health/condition/builder.go +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition - -import ( - "sort" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// skipMergeConditions contain the list of conditions we don't want to add to the list if not recalculated -var skipMergeConditions = map[druidv1alpha1.ConditionType]struct{}{ - druidv1alpha1.ConditionTypeReady: {}, - druidv1alpha1.ConditionTypeAllMembersReady: {}, - druidv1alpha1.ConditionTypeBackupReady: {}, -} - -// Builder is an interface for building conditions. -type Builder interface { - WithOldConditions(conditions []druidv1alpha1.Condition) Builder - WithResults(result []Result) Builder - WithNowFunc(now func() metav1.Time) Builder - Build(replicas int32) []druidv1alpha1.Condition -} - -type defaultBuilder struct { - old map[druidv1alpha1.ConditionType]druidv1alpha1.Condition - results map[druidv1alpha1.ConditionType]Result - nowFunc func() metav1.Time -} - -// NewBuilder returns a Builder for a specific condition. -func NewBuilder() Builder { - return &defaultBuilder{ - old: make(map[druidv1alpha1.ConditionType]druidv1alpha1.Condition), - results: make(map[druidv1alpha1.ConditionType]Result), - nowFunc: func() metav1.Time { - return metav1.NewTime(time.Now().UTC()) - }, - } -} - -// WithOldConditions sets the old conditions. It can be used to provide default values. -func (b *defaultBuilder) WithOldConditions(conditions []druidv1alpha1.Condition) Builder { - for _, cond := range conditions { - b.old[cond.Type] = cond - } - - return b -} - -// WithResults adds the results. -func (b *defaultBuilder) WithResults(results []Result) Builder { - for _, result := range results { - if result == nil { - continue - } - b.results[result.ConditionType()] = result - } - - return b -} - -// WithNowFunc sets the function used for getting the current time. -// Should only be used for tests. -func (b *defaultBuilder) WithNowFunc(now func() metav1.Time) Builder { - b.nowFunc = now - return b -} - -// Build creates the conditions. -// It merges the existing conditions with the results added to the builder. -// If OldCondition is provided: -// - Any changes to status set the `LastTransitionTime` -// - `LastUpdateTime` is always set. -func (b *defaultBuilder) Build(replicas int32) []druidv1alpha1.Condition { - var ( - now = b.nowFunc() - conditions []druidv1alpha1.Condition - ) - - for condType, res := range b.results { - condition, ok := b.old[condType] - if !ok { - condition = druidv1alpha1.Condition{ - Type: condType, - LastTransitionTime: now, - } - } - - if condition.Status != res.Status() { - condition.LastTransitionTime = now - } - condition.LastUpdateTime = now - if replicas == 0 { - if condition.Status == "" { - condition.Status = druidv1alpha1.ConditionUnknown - } - condition.Reason = NotChecked - condition.Message = "etcd cluster has been scaled down" - } else { - condition.Status = res.Status() - condition.Message = res.Message() - condition.Reason = res.Reason() - } - - conditions = append(conditions, condition) - delete(b.old, condType) - } - - for _, condition := range b.old { - // Do not add conditions that are part of the skipMergeConditions list - _, ok := skipMergeConditions[condition.Type] - if ok { - continue - } - // Add existing conditions as they were. This needs to be changed when SSA is used. - conditions = append(conditions, condition) - } - - sort.Slice(conditions, func(i, j int) bool { - return conditions[i].Type < conditions[j].Type - }) - - return conditions -} diff --git a/pkg/health/condition/builder_test.go b/pkg/health/condition/builder_test.go deleted file mode 100644 index 17f5a6da0..000000000 --- a/pkg/health/condition/builder_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition_test - -import ( - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" -) - -var _ = Describe("Builder", func() { - var ( - builder Builder - now time.Time - ) - - BeforeEach(func() { - now, _ = time.Parse(time.RFC3339, "2021-06-01") - builder = NewBuilder() - }) - - JustBeforeEach(func() { - builder.WithNowFunc(func() metav1.Time { - return metav1.NewTime(now) - }) - }) - - Describe("#Build", func() { - Context("when Builder has old conditions", func() { - var ( - oldConditionTime time.Time - oldConditions []druidv1alpha1.Condition - ) - BeforeEach(func() { - oldConditionTime = now.Add(-12 * time.Hour) - - oldConditions = []druidv1alpha1.Condition{ - { - Type: druidv1alpha1.ConditionTypeAllMembersReady, - LastUpdateTime: metav1.NewTime(oldConditionTime), - LastTransitionTime: metav1.NewTime(oldConditionTime), - Status: druidv1alpha1.ConditionTrue, - Reason: "foo reason", - Message: "foo message", - }, - { - Type: druidv1alpha1.ConditionTypeReady, - LastUpdateTime: metav1.NewTime(oldConditionTime), - LastTransitionTime: metav1.NewTime(oldConditionTime), - Status: druidv1alpha1.ConditionFalse, - Reason: "bar reason", - Message: "bar message", - }, - { - Type: druidv1alpha1.ConditionTypeBackupReady, - LastUpdateTime: metav1.NewTime(oldConditionTime), - LastTransitionTime: metav1.NewTime(oldConditionTime), - Status: druidv1alpha1.ConditionTrue, - Reason: "foobar reason", - Message: "foobar message", - }, - } - - builder.WithOldConditions(oldConditions) - }) - - It("should not add old conditions", func() { - builder.WithResults([]Result{ - &result{ - ConType: druidv1alpha1.ConditionTypeAllMembersReady, - ConStatus: druidv1alpha1.ConditionTrue, - ConReason: "new reason", - ConMessage: "new message", - }, - &result{ - ConType: druidv1alpha1.ConditionTypeReady, - ConStatus: druidv1alpha1.ConditionTrue, - ConReason: "new reason", - ConMessage: "new message", - }, - }) - - conditions := builder.Build(1) - - Expect(conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeAllMembersReady), - "LastUpdateTime": Equal(metav1.NewTime(now)), - "LastTransitionTime": Equal(metav1.NewTime(oldConditionTime)), - "Status": Equal(druidv1alpha1.ConditionTrue), - "Reason": Equal("new reason"), - "Message": Equal("new message"), - }), - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeReady), - "LastUpdateTime": Equal(metav1.NewTime(now)), - "LastTransitionTime": Equal(metav1.NewTime(now)), - "Status": Equal(druidv1alpha1.ConditionTrue), - "Reason": Equal("new reason"), - "Message": Equal("new message"), - }), - )) - }) - }) - - Context("when Builder has no old conditions", func() { - It("should correctly set the new conditions", func() { - builder.WithResults([]Result{ - &result{ - ConType: druidv1alpha1.ConditionTypeAllMembersReady, - ConStatus: druidv1alpha1.ConditionTrue, - ConReason: "new reason", - ConMessage: "new message", - }, - &result{ - ConType: druidv1alpha1.ConditionTypeReady, - ConStatus: druidv1alpha1.ConditionTrue, - ConReason: "new reason", - ConMessage: "new message", - }, - }) - - conditions := builder.Build(1) - - Expect(conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeAllMembersReady), - "LastUpdateTime": Equal(metav1.NewTime(now)), - "LastTransitionTime": Equal(metav1.NewTime(now)), - "Status": Equal(druidv1alpha1.ConditionTrue), - "Reason": Equal("new reason"), - "Message": Equal("new message"), - }), - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeReady), - "LastUpdateTime": Equal(metav1.NewTime(now)), - "LastTransitionTime": Equal(metav1.NewTime(now)), - "Status": Equal(druidv1alpha1.ConditionTrue), - "Reason": Equal("new reason"), - "Message": Equal("new message"), - }), - )) - }) - }) - }) -}) - -type result struct { - ConType druidv1alpha1.ConditionType - ConStatus druidv1alpha1.ConditionStatus - ConReason string - ConMessage string -} - -func (r *result) ConditionType() druidv1alpha1.ConditionType { - return r.ConType -} - -func (r *result) Status() druidv1alpha1.ConditionStatus { - return r.ConStatus -} - -func (r *result) Reason() string { - return r.ConReason -} - -func (r *result) Message() string { - return r.ConMessage -} diff --git a/pkg/health/condition/check_all_members.go b/pkg/health/condition/check_all_members.go deleted file mode 100644 index 3368c1b38..000000000 --- a/pkg/health/condition/check_all_members.go +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/go-logr/logr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type allMembersReady struct{} - -func (a *allMembersReady) Check(_ context.Context, _ logr.Logger, etcd druidv1alpha1.Etcd) Result { - if len(etcd.Status.Members) == 0 { - return &result{ - conType: druidv1alpha1.ConditionTypeAllMembersReady, - status: druidv1alpha1.ConditionUnknown, - reason: "NoMembersInStatus", - message: "Cannot determine readiness since status has no members", - } - } - - result := &result{ - conType: druidv1alpha1.ConditionTypeAllMembersReady, - status: druidv1alpha1.ConditionFalse, - reason: "NotAllMembersReady", - message: "At least one member is not ready", - } - - if int32(len(etcd.Status.Members)) < etcd.Spec.Replicas { - // not all members are registered yet - return result - } - - // If we are here this means that all members have registered. Check if any member - // has a not-ready status. If there is at least one then set the overall status as false. - ready := true - for _, member := range etcd.Status.Members { - ready = ready && member.Status == druidv1alpha1.EtcdMemberStatusReady - if !ready { - break - } - } - if ready { - result.status = druidv1alpha1.ConditionTrue - result.reason = "AllMembersReady" - result.message = "All members are ready" - } - - return result -} - -// AllMembersCheck returns a check for the "AllMembersReady" condition. -func AllMembersCheck(_ client.Client) Checker { - return &allMembersReady{} -} diff --git a/pkg/health/condition/check_all_members_test.go b/pkg/health/condition/check_all_members_test.go deleted file mode 100644 index 0d5156e5d..000000000 --- a/pkg/health/condition/check_all_members_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" - "github.com/go-logr/logr" -) - -var _ = Describe("AllMembersReadyCheck", func() { - Describe("#Check", func() { - var ( - readyMember, notReadyMember druidv1alpha1.EtcdMemberStatus - - logger = logr.Discard() - ) - - BeforeEach(func() { - readyMember = druidv1alpha1.EtcdMemberStatus{ - Status: druidv1alpha1.EtcdMemberStatusReady, - } - notReadyMember = druidv1alpha1.EtcdMemberStatus{ - Status: druidv1alpha1.EtcdMemberStatusNotReady, - } - }) - - Context("when members in status", func() { - It("should return that all members are ready", func() { - etcd := druidv1alpha1.Etcd{ - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 3, - }, - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - readyMember, - readyMember, - }, - }, - } - check := AllMembersCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) - }) - - It("should return that members are not ready", func() { - etcd := druidv1alpha1.Etcd{ - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 3, - }, - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - notReadyMember, - readyMember, - }, - }, - } - check := AllMembersCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) - }) - - It("should return all members are not ready when number of members registered are less than spec replicas", func() { - etcd := druidv1alpha1.Etcd{ - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 3, - }, - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - readyMember, - }, - }, - } - check := AllMembersCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) - Expect(result.Reason()).To(Equal("NotAllMembersReady")) - }) - }) - - Context("when no members in status", func() { - It("should return that readiness is unknown", func() { - etcd := druidv1alpha1.Etcd{ - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 3, - }, - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{}, - }, - } - check := AllMembersCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeAllMembersReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) - }) - }) - }) -}) diff --git a/pkg/health/condition/check_backup_ready.go b/pkg/health/condition/check_backup_ready.go deleted file mode 100644 index f72d96ce9..000000000 --- a/pkg/health/condition/check_backup_ready.go +++ /dev/null @@ -1,145 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition - -import ( - "context" - "fmt" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/utils" - - "github.com/go-logr/logr" - coordinationv1 "k8s.io/api/coordination/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type backupReadyCheck struct { - cl client.Client -} - -const ( - // BackupSucceeded is a constant that means that etcd backup has been successfully taken - BackupSucceeded string = "BackupSucceeded" - // BackupFailed is a constant that means that etcd backup has failed - BackupFailed string = "BackupFailed" - // Unknown is a constant that means that the etcd backup status is currently not known - Unknown string = "Unknown" - // NotChecked is a constant that means that the etcd backup status has not been updated or rechecked - NotChecked string = "NotChecked" -) - -func (a *backupReadyCheck) Check(ctx context.Context, logger logr.Logger, etcd druidv1alpha1.Etcd) Result { - //Default case - result := &result{ - conType: druidv1alpha1.ConditionTypeBackupReady, - status: druidv1alpha1.ConditionUnknown, - reason: Unknown, - message: "Cannot determine etcd backup status", - } - - // Special case of etcd not being configured to take snapshots - // Do not add the BackupReady condition if backup is not configured - if etcd.Spec.Backup.Store == nil || etcd.Spec.Backup.Store.Provider == nil || len(*etcd.Spec.Backup.Store.Provider) == 0 { - return nil - } - - // Fetch snapshot leases - var ( - err, fullSnapErr, deltaSnapErr error - fullSnapshotInterval = 1 * time.Hour - fullSnapLease = &coordinationv1.Lease{} - deltaSnapLease = &coordinationv1.Lease{} - ) - fullSnapErr = a.cl.Get(ctx, types.NamespacedName{Name: getFullSnapLeaseName(&etcd), Namespace: etcd.Namespace}, fullSnapLease) - deltaSnapErr = a.cl.Get(ctx, types.NamespacedName{Name: getDeltaSnapLeaseName(&etcd), Namespace: etcd.Namespace}, deltaSnapLease) - - // Set status to Unknown if errors in fetching snapshot leases or lease never renewed - if fullSnapErr != nil || deltaSnapErr != nil || (fullSnapLease.Spec.RenewTime == nil && deltaSnapLease.Spec.RenewTime == nil) { - return result - } - - deltaLeaseRenewTime := deltaSnapLease.Spec.RenewTime - fullLeaseRenewTime := fullSnapLease.Spec.RenewTime - fullLeaseCreateTime := &fullSnapLease.ObjectMeta.CreationTimestamp - - // TODO: make etcd.Spec.Backup.FullSnapshotSchedule non-optional, since it is mandatory to - // set the full snapshot schedule, or introduce defaulting webhook to add default value for this field - if etcd.Spec.Backup.FullSnapshotSchedule != nil { - if fullSnapshotInterval, err = utils.ComputeScheduleInterval(*etcd.Spec.Backup.FullSnapshotSchedule); err != nil { - logger.Error(err, "unable to compute full snapshot duration from full snapshot schedule", "fullSnapshotSchedule", *etcd.Spec.Backup.FullSnapshotSchedule) - return result - } - } - - if fullLeaseRenewTime == nil && deltaLeaseRenewTime != nil { - // Most probable during reconcile of existing clusters if fresh leases are created - // Treat backup as succeeded if delta snap lease renewal happens in the required time window - // and full snap lease is not older than full snapshot duration. - if time.Since(deltaLeaseRenewTime.Time) < 2*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration && time.Since(fullLeaseCreateTime.Time) < fullSnapshotInterval { - result.reason = BackupSucceeded - result.message = "Delta snapshot backup succeeded" - result.status = druidv1alpha1.ConditionTrue - return result - } - } else if deltaLeaseRenewTime == nil && fullLeaseRenewTime != nil { - // Most probable during a startup scenario for new clusters - // Special case. Return Unknown condition for some time to allow delta backups to start up - if time.Since(fullLeaseRenewTime.Time) > 5*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration { - result.message = "Periodic delta snapshots not started yet" - return result - } - } else if deltaLeaseRenewTime != nil && fullLeaseRenewTime != nil { - // Both snap leases are maintained. Both are expected to be renewed periodically - if time.Since(deltaLeaseRenewTime.Time) < 2*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration && time.Since(fullLeaseRenewTime.Time) < fullSnapshotInterval { - result.reason = BackupSucceeded - result.message = "Snapshot backup succeeded" - result.status = druidv1alpha1.ConditionTrue - return result - } - } - - // Cases where snapshot leases are not updated for a long time - // If snapshot leases are present and leases aren't updated, it is safe to assume that backup is not healthy - - if etcd.Status.Conditions != nil { - var prevBackupReadyStatus druidv1alpha1.Condition - for _, prevBackupReadyStatus = range etcd.Status.Conditions { - if prevBackupReadyStatus.Type == druidv1alpha1.ConditionTypeBackupReady { - break - } - } - - // Transition to "False" state only if present state is "Unknown" or "False" - if deltaLeaseRenewTime != nil && (prevBackupReadyStatus.Status == druidv1alpha1.ConditionUnknown || prevBackupReadyStatus.Status == druidv1alpha1.ConditionFalse) { - if time.Since(deltaLeaseRenewTime.Time) > 3*etcd.Spec.Backup.DeltaSnapshotPeriod.Duration { - result.status = druidv1alpha1.ConditionFalse - result.reason = BackupFailed - result.message = "Stale snapshot leases. Not renewed in a long time" - return result - } - } - } - - // Transition to "Unknown" state is we cannot prove a "True" state - return result -} - -func getDeltaSnapLeaseName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-delta-snap", etcd.Name) -} - -func getFullSnapLeaseName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-full-snap", etcd.Name) -} - -// BackupReadyCheck returns a check for the "BackupReady" condition. -func BackupReadyCheck(cl client.Client) Checker { - return &backupReadyCheck{ - cl: cl, - } -} diff --git a/pkg/health/condition/check_backup_ready_test.go b/pkg/health/condition/check_backup_ready_test.go deleted file mode 100644 index c480fb286..000000000 --- a/pkg/health/condition/check_backup_ready_test.go +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition_test - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.uber.org/mock/gomock" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" - - coordinationv1 "k8s.io/api/coordination/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var _ = Describe("BackupReadyCheck", func() { - Describe("#Check", func() { - var ( - storageProvider druidv1alpha1.StorageProvider = "testStorageProvider" - mockCtrl *gomock.Controller - cl *mockclient.MockClient - holderIDString = "123455" - noLeaseError = apierrors.StatusError{ - ErrStatus: v1.Status{ - Reason: v1.StatusReasonNotFound, - }, - } - deltaSnapshotDuration = 2 * time.Minute - logger = ctrl.Log.WithName("backup-ready-checker") - - etcd = druidv1alpha1.Etcd{ - ObjectMeta: v1.ObjectMeta{ - Name: "test-etcd", - Namespace: "default", - }, - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 1, - Backup: druidv1alpha1.BackupSpec{ - FullSnapshotSchedule: pointer.String("0 0 * * *"), // at 00:00 every day - DeltaSnapshotPeriod: &v1.Duration{ - Duration: deltaSnapshotDuration, - }, - Store: &druidv1alpha1.StoreSpec{ - Prefix: "test-prefix", - Provider: &storageProvider, - }, - }, - }, - Status: druidv1alpha1.EtcdStatus{}, - } - lease = coordinationv1.Lease{ - ObjectMeta: v1.ObjectMeta{ - Name: "test-etcd-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: &holderIDString, - RenewTime: &v1.MicroTime{ - Time: time.Now(), - }, - }, - } - ) - - BeforeEach(func() { - mockCtrl = gomock.NewController(GinkgoT()) - cl = mockclient.NewMockClient(mockCtrl) - }) - - AfterEach(func() { - mockCtrl.Finish() - }) - - Context("With no snapshot leases present", func() { - It("Should return Unknown readiness", func() { - cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, er *coordinationv1.Lease, _ ...client.GetOption) error { - return &noLeaseError - }, - ).AnyTimes() - - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).ToNot(BeNil()) - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) - Expect(result.Reason()).To(Equal(Unknown)) - }) - }) - - Context("With both snapshot leases present", func() { - It("Should set status to BackupSucceeded if both leases are recently renewed", func() { - cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - return nil - }, - ).AnyTimes() - - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).ToNot(BeNil()) - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) - Expect(result.Reason()).To(Equal(BackupSucceeded)) - }) - - It("Should set status to BackupSucceeded if delta snap lease is recently created and empty full snap lease has been created recently", func() { - cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-full-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - le.Spec.RenewTime = nil - le.Spec.HolderIdentity = nil - le.ObjectMeta.CreationTimestamp = v1.Now() - return nil - }, - ).AnyTimes() - cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-delta-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - return nil - }, - ).AnyTimes() - - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).ToNot(BeNil()) - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) - Expect(result.Reason()).To(Equal(BackupSucceeded)) - }) - - It("Should set status to Unknown if empty delta snap lease is present but full snap lease is renewed recently", func() { - cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-full-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - le.Spec.RenewTime = &v1.MicroTime{Time: lease.Spec.RenewTime.Time.Add(-5 * deltaSnapshotDuration)} - return nil - }, - ).AnyTimes() - cl.EXPECT().Get(context.TODO(), types.NamespacedName{Name: "test-etcd-delta-snap", Namespace: "default"}, gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - le.Spec.RenewTime = nil - le.Spec.HolderIdentity = nil - return nil - }, - ).AnyTimes() - - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).ToNot(BeNil()) - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) - Expect(result.Reason()).To(Equal(Unknown)) - Expect(result.Message()).To(Equal("Periodic delta snapshots not started yet")) - }) - - It("Should set status to Unknown if both leases are stale", func() { - cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - le.Spec.RenewTime = &v1.MicroTime{ - Time: time.Now().Add(-10 * time.Minute), - } - return nil - }, - ).AnyTimes() - - etcd.Status.Conditions = []druidv1alpha1.Condition{ - { - Type: druidv1alpha1.ConditionTypeBackupReady, - Status: druidv1alpha1.ConditionTrue, - Message: "True", - }, - } - - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).ToNot(BeNil()) - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) - Expect(result.Reason()).To(Equal(Unknown)) - }) - - It("Should set status to BackupFailed if both leases are stale and current condition is Unknown", func() { - cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, le *coordinationv1.Lease, _ ...client.GetOption) error { - *le = lease - le.Spec.RenewTime = &v1.MicroTime{ - Time: time.Now().Add(-10 * time.Minute), - } - return nil - }, - ).AnyTimes() - - etcd.Status.Conditions = []druidv1alpha1.Condition{ - { - Type: druidv1alpha1.ConditionTypeBackupReady, - Status: druidv1alpha1.ConditionUnknown, - Message: "Unknown", - }, - } - - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).ToNot(BeNil()) - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeBackupReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) - Expect(result.Reason()).To(Equal(BackupFailed)) - }) - }) - - Context("With no backup store configured", func() { - It("Should return nil condition", func() { - cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, er *coordinationv1.Lease) error { - return &noLeaseError - }, - ).AnyTimes() - - etcd.Spec.Backup.Store = nil - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).To(BeNil()) - etcd.Spec.Backup.Store = &druidv1alpha1.StoreSpec{ - Prefix: "test-prefix", - Provider: &storageProvider, - } - }) - }) - - Context("With backup store is configured but provider is nil", func() { - It("Should return nil condition", func() { - cl.EXPECT().Get(context.TODO(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, er *coordinationv1.Lease) error { - return &noLeaseError - }, - ).AnyTimes() - - etcd.Spec.Backup.Store.Provider = nil - check := BackupReadyCheck(cl) - result := check.Check(context.TODO(), logger, etcd) - - Expect(result).To(BeNil()) - etcd.Spec.Backup.Store.Provider = &storageProvider - }) - }) - }) -}) diff --git a/pkg/health/condition/check_ready.go b/pkg/health/condition/check_ready.go deleted file mode 100644 index 0ead3cde0..000000000 --- a/pkg/health/condition/check_ready.go +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/go-logr/logr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type readyCheck struct{} - -func (r *readyCheck) Check(_ context.Context, _ logr.Logger, etcd druidv1alpha1.Etcd) Result { - - // TODO: remove this case as soon as leases are completely supported by etcd-backup-restore - if len(etcd.Status.Members) == 0 { - return &result{ - conType: druidv1alpha1.ConditionTypeReady, - status: druidv1alpha1.ConditionUnknown, - reason: "NoMembersInStatus", - message: "Cannot determine readiness since status has no members", - } - } - - var ( - size = len(etcd.Status.Members) - quorum = size/2 + 1 - readyMembers = 0 - ) - - for _, member := range etcd.Status.Members { - if member.Status == druidv1alpha1.EtcdMemberStatusNotReady { - continue - } - readyMembers++ - } - - if readyMembers < quorum { - return &result{ - conType: druidv1alpha1.ConditionTypeReady, - status: druidv1alpha1.ConditionFalse, - reason: "QuorumLost", - message: "The majority of ETCD members is not ready", - } - } - - return &result{ - conType: druidv1alpha1.ConditionTypeReady, - status: druidv1alpha1.ConditionTrue, - reason: "Quorate", - message: "The majority of ETCD members is ready", - } -} - -// ReadyCheck returns a check for the "Ready" condition. -func ReadyCheck(_ client.Client) Checker { - return &readyCheck{} -} diff --git a/pkg/health/condition/check_ready_test.go b/pkg/health/condition/check_ready_test.go deleted file mode 100644 index 7af7be8a7..000000000 --- a/pkg/health/condition/check_ready_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/condition" - "github.com/go-logr/logr" -) - -var _ = Describe("ReadyCheck", func() { - Describe("#Check", func() { - var ( - readyMember, notReadyMember, unknownMember druidv1alpha1.EtcdMemberStatus - - logger = logr.Discard() - ) - - BeforeEach(func() { - readyMember = druidv1alpha1.EtcdMemberStatus{ - Status: druidv1alpha1.EtcdMemberStatusReady, - } - notReadyMember = druidv1alpha1.EtcdMemberStatus{ - Status: druidv1alpha1.EtcdMemberStatusNotReady, - } - unknownMember = druidv1alpha1.EtcdMemberStatus{ - Status: druidv1alpha1.EtcdMemberStatusUnknown, - } - }) - - Context("when members in status", func() { - It("should return that the cluster has a quorum (all members ready)", func() { - etcd := druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - readyMember, - readyMember, - }, - }, - } - check := ReadyCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) - }) - - It("should return that the cluster has a quorum (members are partly unknown)", func() { - etcd := druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - unknownMember, - unknownMember, - }, - }, - } - check := ReadyCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) - }) - - It("should return that the cluster has a quorum (one member not ready)", func() { - etcd := druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - notReadyMember, - readyMember, - }, - }, - } - check := ReadyCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) - }) - - It("should return that the cluster has lost its quorum", func() { - etcd := druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{ - readyMember, - notReadyMember, - notReadyMember, - }, - }, - } - check := ReadyCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) - Expect(result.Reason()).To(Equal("QuorumLost")) - }) - }) - - Context("when no members in status", func() { - It("should return that quorum is unknown", func() { - etcd := druidv1alpha1.Etcd{ - Status: druidv1alpha1.EtcdStatus{ - Members: []druidv1alpha1.EtcdMemberStatus{}, - }, - } - check := ReadyCheck(nil) - - result := check.Check(context.TODO(), logger, etcd) - - Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeReady)) - Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) - Expect(result.Reason()).To(Equal("NoMembersInStatus")) - }) - }) - }) -}) diff --git a/pkg/health/condition/condition_suite_test.go b/pkg/health/condition/condition_suite_test.go deleted file mode 100644 index 7eb295e3e..000000000 --- a/pkg/health/condition/condition_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestCondition(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Condition Suite") -} diff --git a/pkg/health/condition/types.go b/pkg/health/condition/types.go deleted file mode 100644 index 0562aec90..000000000 --- a/pkg/health/condition/types.go +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package condition - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/go-logr/logr" -) - -// Checker is an interface to check the etcd resource and to return condition results. -type Checker interface { - Check(ctx context.Context, logger logr.Logger, etcd druidv1alpha1.Etcd) Result -} - -// Result encapsulates a condition result -type Result interface { - ConditionType() druidv1alpha1.ConditionType - Status() druidv1alpha1.ConditionStatus - Reason() string - Message() string -} - -type result struct { - conType druidv1alpha1.ConditionType - status druidv1alpha1.ConditionStatus - reason string - message string -} - -func (r *result) ConditionType() druidv1alpha1.ConditionType { - return r.conType -} - -func (r *result) Status() druidv1alpha1.ConditionStatus { - return r.status -} - -func (r *result) Reason() string { - return r.reason -} - -func (r *result) Message() string { - return r.message -} diff --git a/pkg/health/etcdmember/builder.go b/pkg/health/etcdmember/builder.go deleted file mode 100644 index eb643f2a2..000000000 --- a/pkg/health/etcdmember/builder.go +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdmember - -import ( - "sort" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Builder is an interface for building status objects for etcd members. -type Builder interface { - WithOldMembers(members []druidv1alpha1.EtcdMemberStatus) Builder - WithResults(results []Result) Builder - WithNowFunc(now func() metav1.Time) Builder - Build() []druidv1alpha1.EtcdMemberStatus -} - -type defaultBuilder struct { - old map[string]druidv1alpha1.EtcdMemberStatus - results map[string]Result - nowFunc func() metav1.Time -} - -// NewBuilder returns a Builder for a specific etcd member status. -func NewBuilder() Builder { - return &defaultBuilder{ - old: make(map[string]druidv1alpha1.EtcdMemberStatus), - results: make(map[string]Result), - nowFunc: func() metav1.Time { - return metav1.NewTime(time.Now().UTC()) - }, - } -} - -// WithOldMember sets the old etcd member statuses. It can be used to provide default values. -func (b *defaultBuilder) WithOldMembers(members []druidv1alpha1.EtcdMemberStatus) Builder { - for _, member := range members { - b.old[member.Name] = member - } - - return b -} - -// WithResults adds the results. -func (b *defaultBuilder) WithResults(results []Result) Builder { - for _, res := range results { - if res == nil { - continue - } - b.results[res.Name()] = res - } - - return b -} - -// WithNowFunc sets the function used for getting the current time. -// Should only be used for tests. -func (b *defaultBuilder) WithNowFunc(now func() metav1.Time) Builder { - b.nowFunc = now - return b -} - -// Build creates the etcd member statuses. -// It merges the existing members with the results added to the builder. -// If OldCondition is provided: -// - Any changes to status set the `LastTransitionTime` -func (b *defaultBuilder) Build() []druidv1alpha1.EtcdMemberStatus { - var ( - now = b.nowFunc() - - members []druidv1alpha1.EtcdMemberStatus - ) - - for name, res := range b.results { - memberStatus := druidv1alpha1.EtcdMemberStatus{ - ID: res.ID(), - Name: res.Name(), - Role: res.Role(), - Status: res.Status(), - Reason: res.Reason(), - LastTransitionTime: now, - } - - // Don't reset LastTransitionTime if status didn't change - if oldMemberStatus, ok := b.old[name]; ok { - if oldMemberStatus.Status == res.Status() { - memberStatus.LastTransitionTime = oldMemberStatus.LastTransitionTime - } - } - - members = append(members, memberStatus) - } - - sort.Slice(members, func(i, j int) bool { - return members[i].Name < members[j].Name - }) - - return members -} diff --git a/pkg/health/etcdmember/builder_test.go b/pkg/health/etcdmember/builder_test.go deleted file mode 100644 index 546f68bb0..000000000 --- a/pkg/health/etcdmember/builder_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdmember_test - -import ( - "time" - - "k8s.io/utils/pointer" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/health/etcdmember" -) - -var _ = Describe("Builder", func() { - var ( - builder Builder - now time.Time - ) - - BeforeEach(func() { - now, _ = time.Parse(time.RFC3339, "2021-06-01") - builder = NewBuilder() - }) - - JustBeforeEach(func() { - builder.WithNowFunc(func() metav1.Time { - return metav1.NewTime(now) - }) - }) - - Describe("#Build", func() { - Context("when Builder has old members", func() { - var ( - oldMembers map[string]druidv1alpha1.EtcdMemberStatus - ) - BeforeEach(func() { - oldMembers = map[string]druidv1alpha1.EtcdMemberStatus{ - "1": { - Name: "member1", - ID: pointer.String("1"), - Status: druidv1alpha1.EtcdMemberStatusReady, - Reason: "foo reason", - LastTransitionTime: metav1.NewTime(now.Add(-12 * time.Hour)), - }, - "2": { - Name: "member2", - ID: pointer.String("2"), - Status: druidv1alpha1.EtcdMemberStatusReady, - Reason: "bar reason", - LastTransitionTime: metav1.NewTime(now.Add(-6 * time.Hour)), - }, - "3": { - Name: "member3", - ID: pointer.String("3"), - Status: druidv1alpha1.EtcdMemberStatusReady, - Reason: "foobar reason", - LastTransitionTime: metav1.NewTime(now.Add(-18 * time.Hour)), - }, - } - - builder.WithOldMembers([]druidv1alpha1.EtcdMemberStatus{ - oldMembers["1"], - oldMembers["2"], - oldMembers["3"], - }) - }) - - It("should correctly set the LastTransitionTime", func() { - builder.WithResults([]Result{ - &result{ - MemberID: pointer.String("3"), - MemberName: "member3", - MemberStatus: druidv1alpha1.EtcdMemberStatusUnknown, - MemberReason: "unknown reason", - }, - }) - - conditions := builder.Build() - - Expect(conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Name": Equal("member3"), - "ID": PointTo(Equal("3")), - "Status": Equal(druidv1alpha1.EtcdMemberStatusUnknown), - "Reason": Equal("unknown reason"), - "LastTransitionTime": Equal(metav1.NewTime(now)), - }), - )) - }) - }) - - Context("when Builder has no old members", func() { - var ( - memberRoleLeader, memberRoleMember druidv1alpha1.EtcdRole - ) - - BeforeEach(func() { - memberRoleLeader = druidv1alpha1.EtcdRoleLeader - memberRoleMember = druidv1alpha1.EtcdRoleMember - }) - - It("should not add any members but sort them", func() { - builder.WithResults([]Result{ - &result{ - MemberID: pointer.String("2"), - MemberName: "member2", - MemberRole: &memberRoleMember, - MemberStatus: druidv1alpha1.EtcdMemberStatusReady, - MemberReason: "foo reason", - }, - &result{ - MemberID: pointer.String("1"), - MemberName: "member1", - MemberRole: &memberRoleLeader, - MemberStatus: druidv1alpha1.EtcdMemberStatusUnknown, - MemberReason: "unknown reason", - }, - }) - - conditions := builder.Build() - - Expect(conditions).To(HaveLen(2)) - Expect(conditions[0]).To(MatchFields(IgnoreExtras, Fields{ - "Name": Equal("member1"), - "ID": PointTo(Equal("1")), - "Role": PointTo(Equal(druidv1alpha1.EtcdRoleLeader)), - "Status": Equal(druidv1alpha1.EtcdMemberStatusUnknown), - "Reason": Equal("unknown reason"), - "LastTransitionTime": Equal(metav1.NewTime(now)), - })) - Expect(conditions[1]).To(MatchFields(IgnoreExtras, Fields{ - "Name": Equal("member2"), - "ID": PointTo(Equal("2")), - "Role": PointTo(Equal(druidv1alpha1.EtcdRoleMember)), - "Status": Equal(druidv1alpha1.EtcdMemberStatusReady), - "Reason": Equal("foo reason"), - "LastTransitionTime": Equal(metav1.NewTime(now)), - })) - }) - }) - }) -}) - -type result struct { - MemberID *string - MemberName string - MemberRole *druidv1alpha1.EtcdRole - MemberStatus druidv1alpha1.EtcdMemberConditionStatus - MemberReason string -} - -func (r *result) ID() *string { - return r.MemberID -} - -func (r *result) Name() string { - return r.MemberName -} - -func (r *result) Role() *druidv1alpha1.EtcdRole { - return r.MemberRole -} - -func (r *result) Reason() string { - return r.MemberReason -} - -func (r *result) Status() druidv1alpha1.EtcdMemberConditionStatus { - return r.MemberStatus -} diff --git a/pkg/health/etcdmember/check_ready.go b/pkg/health/etcdmember/check_ready.go deleted file mode 100644 index 05cd7f936..000000000 --- a/pkg/health/etcdmember/check_ready.go +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdmember - -import ( - "context" - "strings" - "time" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" -) - -type readyCheck struct { - logger logr.Logger - cl client.Client - etcdMemberNotReadyThreshold time.Duration - etcdMemberUnknownThreshold time.Duration -} - -// TimeNow is the function used by this check to get the current time. -var TimeNow = time.Now - -func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Result { - var ( - results []Result - checkTime = TimeNow().UTC() - ) - - leases := &coordinationv1.LeaseList{} - if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ - common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: utils.PurposeMemberLease}); err != nil { - r.logger.Error(err, "failed to get leases for etcd member readiness check") - } - - for _, lease := range leases.Items { - var ( - id, role = separateIdFromRole(lease.Spec.HolderIdentity) - res = &result{ - id: id, - name: lease.Name, - role: role, - } - ) - - // Check if member is in bootstrapping phase - // Members are supposed to be added to the members array only if they have joined the cluster (== RenewTime is set). - // This behavior is expected by the `Ready` condition and it will become imprecise if members are added here too early. - renew := lease.Spec.RenewTime - if renew == nil { - r.logger.Info("Member hasn't acquired lease yet, still in bootstrapping phase", "name", lease.Name) - continue - } - - // Check if member state must be considered as not ready - if renew.Add(r.etcdMemberUnknownThreshold).Add(r.etcdMemberNotReadyThreshold).Before(checkTime) { - res.status = druidv1alpha1.EtcdMemberStatusNotReady - res.reason = "UnknownGracePeriodExceeded" - results = append(results, res) - continue - } - - // Check if member state must be considered as unknown - if renew.Add(r.etcdMemberUnknownThreshold).Before(checkTime) { - // If pod is not running or cannot be found then we deduce that the status is NotReady. - ready, err := r.checkContainersAreReady(ctx, lease.Namespace, lease.Name) - if (err == nil && !ready) || apierrors.IsNotFound(err) { - res.status = druidv1alpha1.EtcdMemberStatusNotReady - res.reason = "ContainersNotReady" - results = append(results, res) - continue - } - - res.status = druidv1alpha1.EtcdMemberStatusUnknown - res.reason = "LeaseExpired" - results = append(results, res) - continue - } - - res.status = druidv1alpha1.EtcdMemberStatusReady - res.reason = "LeaseSucceeded" - results = append(results, res) - } - - return results -} - -const holderIdentitySeparator = ":" - -func separateIdFromRole(holderIdentity *string) (*string, *druidv1alpha1.EtcdRole) { - if holderIdentity == nil { - return nil, nil - } - parts := strings.SplitN(*holderIdentity, holderIdentitySeparator, 2) - id := &parts[0] - if len(parts) != 2 { - return id, nil - } - - switch druidv1alpha1.EtcdRole(parts[1]) { - case druidv1alpha1.EtcdRoleLeader: - role := druidv1alpha1.EtcdRoleLeader - return id, &role - case druidv1alpha1.EtcdRoleMember: - role := druidv1alpha1.EtcdRoleMember - return id, &role - default: - return id, nil - } -} - -func (r *readyCheck) checkContainersAreReady(ctx context.Context, namespace string, name string) (bool, error) { - pod := &corev1.Pod{} - if err := r.cl.Get(ctx, kutil.Key(namespace, name), pod); err != nil { - return false, err - } - - for _, cond := range pod.Status.Conditions { - if cond.Type == corev1.ContainersReady { - return cond.Status == corev1.ConditionTrue, nil - } - } - - return false, nil -} - -// ReadyCheck returns a check for the "Ready" condition. -func ReadyCheck(cl client.Client, logger logr.Logger, etcdMemberNotReadyThreshold, etcdMemberUnknownThreshold time.Duration) Checker { - return &readyCheck{ - logger: logger, - cl: cl, - etcdMemberNotReadyThreshold: etcdMemberNotReadyThreshold, - etcdMemberUnknownThreshold: etcdMemberUnknownThreshold, - } -} diff --git a/pkg/health/etcdmember/check_ready_test.go b/pkg/health/etcdmember/check_ready_test.go deleted file mode 100644 index 74a8efc87..000000000 --- a/pkg/health/etcdmember/check_ready_test.go +++ /dev/null @@ -1,393 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdmember_test - -import ( - "context" - "errors" - "fmt" - "time" - - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/go-logr/logr" - - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/gardener/gardener/pkg/utils/test" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - "go.uber.org/mock/gomock" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - . "github.com/gardener/etcd-druid/pkg/health/etcdmember" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" - "github.com/gardener/etcd-druid/pkg/utils" -) - -var _ = Describe("ReadyCheck", func() { - Describe("#Check", func() { - var ( - ctx context.Context - mockCtrl *gomock.Controller - cl *mockclient.MockClient - leaseDurationSeconds *int32 - unknownThreshold, notReadyThreshold time.Duration - now time.Time - check Checker - logger logr.Logger - - member1Name string - member1ID *string - etcd druidv1alpha1.Etcd - leasesList *coordinationv1.LeaseList - ) - - BeforeEach(func() { - ctx = context.Background() - mockCtrl = gomock.NewController(GinkgoT()) - cl = mockclient.NewMockClient(mockCtrl) - unknownThreshold = 300 * time.Second - notReadyThreshold = 60 * time.Second - now, _ = time.Parse(time.RFC3339, "2021-06-01T00:00:00Z") - logger = log.Log.WithName("Test") - check = ReadyCheck(cl, logger, notReadyThreshold, unknownThreshold) - - member1ID = pointer.String("1") - member1Name = "member1" - - etcd = druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd", - Namespace: "etcd-test", - }, - } - }) - - AfterEach(func() { - mockCtrl.Finish() - }) - - JustBeforeEach(func() { - cl.EXPECT().List(ctx, gomock.AssignableToTypeOf(&coordinationv1.LeaseList{}), client.InNamespace(etcd.Namespace), - client.MatchingLabels{common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: utils.PurposeMemberLease}). - DoAndReturn( - func(_ context.Context, leases *coordinationv1.LeaseList, _ ...client.ListOption) error { - *leases = *leasesList - return nil - }) - }) - - Context("when just expired", func() { - BeforeEach(func() { - renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second)) - leasesList = &coordinationv1.LeaseList{ - Items: []coordinationv1.Lease{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: member1Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &renewTime, - }, - }, - }, - } - }) - - It("should set the affected condition to UNKNOWN because lease is lost", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { - *pod = corev1.Pod{ - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.ContainersReady, - Status: corev1.ConditionTrue, - }, - }, - }, - } - return nil - }, - ) - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(1)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusUnknown)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - }) - - It("should set the affected condition to UNKNOWN because Pod cannot be received", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { - return errors.New("foo") - }, - ) - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(1)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusUnknown)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - }) - - It("should set the affected condition to FAILED because containers are not ready", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { - *pod = corev1.Pod{ - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.ContainersReady, - Status: corev1.ConditionFalse, - }, - }, - }, - } - return nil - }, - ) - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(1)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusNotReady)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - }) - - It("should set the affected condition to FAILED because Pod is not found", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { - return apierrors.NewNotFound(corev1.Resource("pods"), member1Name) - }, - ) - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(1)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusNotReady)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - }) - }) - - Context("when expired a while ago", func() { - var ( - member2Name string - member2ID *string - ) - - BeforeEach(func() { - member2Name = "member2" - member2ID = pointer.String("2") - - var ( - shortExpirationTime = metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second)) - longExpirationTime = metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second).Add(-1 * notReadyThreshold)) - ) - - leasesList = &coordinationv1.LeaseList{ - Items: []coordinationv1.Lease{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: member1Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &shortExpirationTime, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: member2Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member2ID, druidv1alpha1.EtcdRoleMember)), - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &longExpirationTime, - }, - }, - }, - } - }) - - It("should set the affected condition to FAILED because status was Unknown for a while", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - cl.EXPECT().Get(ctx, kutil.Key(etcd.Namespace, member1Name), gomock.AssignableToTypeOf(&corev1.Pod{})).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, pod *corev1.Pod, _ ...client.ListOption) error { - return errors.New("foo") - }, - ) - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(2)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusUnknown)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - Expect(results[1].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusNotReady)) - Expect(results[1].ID()).To(Equal(member2ID)) - Expect(results[1].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleMember))) - }) - }) - - Context("when lease is up-to-date", func() { - var ( - member2Name, member3Name string - member2ID, member3ID *string - ) - - BeforeEach(func() { - member2Name = "member2" - member2ID = pointer.String("2") - member3Name = "member3" - member3ID = pointer.String("3") - renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold)) - leasesList = &coordinationv1.LeaseList{ - Items: []coordinationv1.Lease{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: member1Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &renewTime, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: member2Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: member2ID, - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &renewTime, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: member3Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member3ID, "foo")), - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &renewTime, - }, - }, - }, - } - }) - - It("should set member ready", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(3)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - Expect(results[1].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) - Expect(results[1].ID()).To(Equal(member2ID)) - Expect(results[1].Role()).To(BeNil()) - Expect(results[2].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) - Expect(results[2].ID()).To(Equal(member3ID)) - Expect(results[2].Role()).To(BeNil()) - }) - }) - - Context("when lease has not been acquired", func() { - var ( - member2Name string - ) - - BeforeEach(func() { - member2Name = "member2" - renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold)) - leasesList = &coordinationv1.LeaseList{ - Items: []coordinationv1.Lease{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: member1Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String(fmt.Sprintf("%s:%s", *member1ID, druidv1alpha1.EtcdRoleLeader)), - LeaseDurationSeconds: leaseDurationSeconds, - RenewTime: &renewTime, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: member2Name, - Namespace: etcd.Namespace, - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("foo"), - LeaseDurationSeconds: leaseDurationSeconds, - }, - }, - }, - } - }) - - It("should only contain members which acquired lease once", func() { - defer test.WithVar(&TimeNow, func() time.Time { - return now - })() - - results := check.Check(ctx, etcd) - - Expect(results).To(HaveLen(1)) - Expect(results[0].Status()).To(Equal(druidv1alpha1.EtcdMemberStatusReady)) - Expect(results[0].ID()).To(Equal(member1ID)) - Expect(results[0].Role()).To(gstruct.PointTo(Equal(druidv1alpha1.EtcdRoleLeader))) - }) - }) - }) -}) diff --git a/pkg/health/etcdmember/etcdmember_suite_test.go b/pkg/health/etcdmember/etcdmember_suite_test.go deleted file mode 100644 index 9be90ecf9..000000000 --- a/pkg/health/etcdmember/etcdmember_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdmember_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestEtcdMember(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Etcd Member Suite") -} diff --git a/pkg/health/etcdmember/types.go b/pkg/health/etcdmember/types.go deleted file mode 100644 index 7731bedd1..000000000 --- a/pkg/health/etcdmember/types.go +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcdmember - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" -) - -// Checker is an interface to check the members of an etcd cluster. -type Checker interface { - Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Result -} - -// Result is an interface to capture the result of checks on etcd members. -type Result interface { - ID() *string - Name() string - Role() *druidv1alpha1.EtcdRole - Status() druidv1alpha1.EtcdMemberConditionStatus - Reason() string -} - -type result struct { - id *string - name string - role *druidv1alpha1.EtcdRole - status druidv1alpha1.EtcdMemberConditionStatus - reason string -} - -func (r *result) ID() *string { - return r.id -} - -func (r *result) Name() string { - return r.name -} - -func (r *result) Role() *druidv1alpha1.EtcdRole { - return r.role -} - -func (r *result) Status() druidv1alpha1.EtcdMemberConditionStatus { - return r.status -} - -func (r *result) Reason() string { - return r.reason -} diff --git a/pkg/health/status/check.go b/pkg/health/status/check.go deleted file mode 100644 index cfee8334a..000000000 --- a/pkg/health/status/check.go +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package status - -import ( - "context" - "sync" - "time" - - "github.com/go-logr/logr" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/health/condition" - "github.com/gardener/etcd-druid/pkg/health/etcdmember" -) - -// ConditionCheckFn is a type alias for a function which returns an implementation of `Check`. -type ConditionCheckFn func(client.Client) condition.Checker - -// EtcdMemberCheckFn is a type alias for a function which returns an implementation of `Check`. -type EtcdMemberCheckFn func(client.Client, logr.Logger, time.Duration, time.Duration) etcdmember.Checker - -// TimeNow is the function used to get the current time. -var TimeNow = time.Now - -var ( - // NewDefaultConditionBuilder is the default condition builder. - NewDefaultConditionBuilder = condition.NewBuilder - // NewDefaultEtcdMemberBuilder is the default etcd member builder. - NewDefaultEtcdMemberBuilder = etcdmember.NewBuilder - // ConditionChecks Checks are the registered condition checks. - ConditionChecks = []ConditionCheckFn{ - condition.AllMembersCheck, - condition.ReadyCheck, - condition.BackupReadyCheck, - } - // EtcdMemberChecks are the etcd member checks. - EtcdMemberChecks = []EtcdMemberCheckFn{ - etcdmember.ReadyCheck, - } -) - -// Checker checks Etcd status conditions and the status of the Etcd members. -type Checker struct { - cl client.Client - etcdMemberNotReadyThreshold time.Duration - etcdMemberUnknownThreshold time.Duration - conditionCheckFns []ConditionCheckFn - conditionBuilderFn func() condition.Builder - etcdMemberCheckFns []EtcdMemberCheckFn - etcdMemberBuilderFn func() etcdmember.Builder -} - -// Check executes the status checks and mutates the passed status object with the corresponding results. -func (c *Checker) Check(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { - // First execute the etcd member checks for the status. - if err := c.executeEtcdMemberChecks(ctx, logger, etcd); err != nil { - return err - } - - // Execute condition checks after the etcd member checks because we need their result here. - return c.executeConditionChecks(ctx, logger, etcd) -} - -// executeConditionChecks runs all registered condition checks **in parallel**. -func (c *Checker) executeConditionChecks(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { - var ( - resultCh = make(chan condition.Result) - - wg sync.WaitGroup - ) - - // Run condition checks in parallel since they work independent of each other. - for _, newCheck := range c.conditionCheckFns { - c := newCheck(c.cl) - wg.Add(1) - go (func() { - defer wg.Done() - resultCh <- c.Check(ctx, logger, *etcd) - })() - } - - go (func() { - defer close(resultCh) - wg.Wait() - })() - - results := make([]condition.Result, 0, len(ConditionChecks)) - for r := range resultCh { - results = append(results, r) - } - - conditions := c.conditionBuilderFn(). - WithNowFunc(func() metav1.Time { return metav1.NewTime(TimeNow()) }). - WithOldConditions(etcd.Status.Conditions). - WithResults(results). - Build(etcd.Spec.Replicas) - - etcd.Status.Conditions = conditions - return nil -} - -// executeEtcdMemberChecks runs all registered etcd member checks **sequentially**. -// The result of a check is passed via the `status` sub-resources to the next check. -func (c *Checker) executeEtcdMemberChecks(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { - // Run etcd member checks sequentially as most of them act on multiple elements. - for _, newCheck := range c.etcdMemberCheckFns { - results := newCheck(c.cl, logger, c.etcdMemberNotReadyThreshold, c.etcdMemberUnknownThreshold).Check(ctx, *etcd) - - // Build and assign the results after each check, so that the next check - // can act on the latest results. - memberStatuses := c.etcdMemberBuilderFn(). - WithNowFunc(func() metav1.Time { return metav1.NewTime(TimeNow()) }). - WithOldMembers(etcd.Status.Members). - WithResults(results). - Build() - - etcd.Status.Members = memberStatuses - } - return nil -} - -// NewChecker creates a new instance for checking the etcd status. -func NewChecker(cl client.Client, etcdMemberNotReadyThreshold, etcdMemberUnknownThreshold time.Duration) *Checker { - return &Checker{ - cl: cl, - etcdMemberNotReadyThreshold: etcdMemberNotReadyThreshold, - etcdMemberUnknownThreshold: etcdMemberUnknownThreshold, - conditionCheckFns: ConditionChecks, - conditionBuilderFn: NewDefaultConditionBuilder, - etcdMemberCheckFns: EtcdMemberChecks, - etcdMemberBuilderFn: NewDefaultEtcdMemberBuilder, - } -} diff --git a/pkg/health/status/check_test.go b/pkg/health/status/check_test.go deleted file mode 100644 index 63153caf7..000000000 --- a/pkg/health/status/check_test.go +++ /dev/null @@ -1,271 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package status_test - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/health/condition" - "github.com/gardener/etcd-druid/pkg/health/etcdmember" - . "github.com/gardener/etcd-druid/pkg/health/status" - - "github.com/gardener/gardener/pkg/utils/test" - "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -var _ = Describe("Check", func() { - Describe("#Check", func() { - It("should correctly execute checks and fill status", func() { - memberRoleLeader := druidv1alpha1.EtcdRoleLeader - memberRoleMember := druidv1alpha1.EtcdRoleMember - - timeBefore, _ := time.Parse(time.RFC3339, "2021-06-01T00:00:00Z") - timeNow := timeBefore.Add(1 * time.Hour) - - status := druidv1alpha1.EtcdStatus{ - Conditions: []druidv1alpha1.Condition{ - { - Type: druidv1alpha1.ConditionTypeReady, - Status: druidv1alpha1.ConditionTrue, - LastTransitionTime: metav1.NewTime(timeBefore), - LastUpdateTime: metav1.NewTime(timeBefore), - Reason: "foo reason", - Message: "foo message", - }, - { - Type: druidv1alpha1.ConditionTypeAllMembersReady, - Status: druidv1alpha1.ConditionTrue, - LastTransitionTime: metav1.NewTime(timeBefore), - LastUpdateTime: metav1.NewTime(timeBefore), - Reason: "bar reason", - Message: "bar message", - }, - { - Type: druidv1alpha1.ConditionTypeBackupReady, - Status: druidv1alpha1.ConditionUnknown, - LastTransitionTime: metav1.NewTime(timeBefore), - LastUpdateTime: metav1.NewTime(timeBefore), - Reason: "foobar reason", - Message: "foobar message", - }, - }, - Members: []druidv1alpha1.EtcdMemberStatus{ - { - ID: pointer.String("1"), - Name: "member1", - Status: druidv1alpha1.EtcdMemberStatusReady, - LastTransitionTime: metav1.NewTime(timeBefore), - Reason: "foo reason", - }, - { - ID: pointer.String("2"), - Name: "member2", - Status: druidv1alpha1.EtcdMemberStatusNotReady, - LastTransitionTime: metav1.NewTime(timeBefore), - Reason: "bar reason", - }, - { - ID: pointer.String("3"), - Name: "member3", - Status: druidv1alpha1.EtcdMemberStatusReady, - LastTransitionTime: metav1.NewTime(timeBefore), - Reason: "foobar reason", - }, - }, - } - - etcd := &druidv1alpha1.Etcd{ - Spec: druidv1alpha1.EtcdSpec{ - Replicas: 1, - }, - Status: status, - } - - defer test.WithVar(&ConditionChecks, []ConditionCheckFn{ - func(client.Client) condition.Checker { - return createConditionCheck(druidv1alpha1.ConditionTypeReady, druidv1alpha1.ConditionFalse, "FailedConditionCheck", "check failed") - }, - func(client.Client) condition.Checker { - return createConditionCheck(druidv1alpha1.ConditionTypeAllMembersReady, druidv1alpha1.ConditionTrue, "bar reason", "bar message") - }, - func(client.Client) condition.Checker { - return createConditionCheck(druidv1alpha1.ConditionTypeBackupReady, druidv1alpha1.ConditionUnknown, "foobar reason", "foobar message") - }, - })() - - defer test.WithVar(&EtcdMemberChecks, []EtcdMemberCheckFn{ - func(_ client.Client, _ logr.Logger, _, _ time.Duration) etcdmember.Checker { - return createEtcdMemberCheck( - etcdMemberResult{pointer.String("1"), "member1", &memberRoleLeader, druidv1alpha1.EtcdMemberStatusUnknown, "Unknown"}, - etcdMemberResult{pointer.String("2"), "member2", &memberRoleMember, druidv1alpha1.EtcdMemberStatusNotReady, "bar reason"}, - etcdMemberResult{pointer.String("3"), "member3", &memberRoleMember, druidv1alpha1.EtcdMemberStatusReady, "foobar reason"}, - ) - }, - })() - - defer test.WithVar(&TimeNow, func() time.Time { return timeNow })() - - checker := NewChecker(nil, 5*time.Minute, time.Minute) - logger := log.Log.WithName("Test") - - Expect(checker.Check(context.Background(), logger, etcd)).To(Succeed()) - - Expect(etcd.Status.Conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeReady), - "Status": Equal(druidv1alpha1.ConditionFalse), - "LastTransitionTime": Equal(metav1.NewTime(timeNow)), - "LastUpdateTime": Equal(metav1.NewTime(timeNow)), - "Reason": Equal("FailedConditionCheck"), - "Message": Equal("check failed"), - }), - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeAllMembersReady), - "Status": Equal(druidv1alpha1.ConditionTrue), - "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), - "LastUpdateTime": Equal(metav1.NewTime(timeNow)), - "Reason": Equal("bar reason"), - "Message": Equal("bar message"), - }), - MatchFields(IgnoreExtras, Fields{ - "Type": Equal(druidv1alpha1.ConditionTypeBackupReady), - "Status": Equal(druidv1alpha1.ConditionUnknown), - "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), - "LastUpdateTime": Equal(metav1.NewTime(timeNow)), - "Reason": Equal("foobar reason"), - "Message": Equal("foobar message"), - }), - )) - - Expect(etcd.Status.Members).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "ID": PointTo(Equal("1")), - "Name": Equal("member1"), - "Role": PointTo(Equal(druidv1alpha1.EtcdRoleLeader)), - "Status": Equal(druidv1alpha1.EtcdMemberStatusUnknown), - "LastTransitionTime": Equal(metav1.NewTime(timeNow)), - "Reason": Equal("Unknown"), - }), - MatchFields(IgnoreExtras, Fields{ - "ID": PointTo(Equal("2")), - "Name": Equal("member2"), - "Role": PointTo(Equal(druidv1alpha1.EtcdRoleMember)), - "Status": Equal(druidv1alpha1.EtcdMemberStatusNotReady), - "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), - "Reason": Equal("bar reason"), - }), - MatchFields(IgnoreExtras, Fields{ - "ID": PointTo(Equal("3")), - "Name": Equal("member3"), - "Role": PointTo(Equal(druidv1alpha1.EtcdRoleMember)), - "Status": Equal(druidv1alpha1.EtcdMemberStatusReady), - "LastTransitionTime": Equal(metav1.NewTime(timeBefore)), - "Reason": Equal("foobar reason"), - }), - )) - - }) - }) -}) - -type conditionResult struct { - ConType druidv1alpha1.ConditionType - ConStatus druidv1alpha1.ConditionStatus - ConReason string - ConMessage string -} - -func (r *conditionResult) ConditionType() druidv1alpha1.ConditionType { - return r.ConType -} - -func (r *conditionResult) Status() druidv1alpha1.ConditionStatus { - return r.ConStatus -} - -func (r *conditionResult) Reason() string { - return r.ConReason -} - -func (r *conditionResult) Message() string { - return r.ConMessage -} - -type conditionTestChecker struct { - result *conditionResult -} - -func (t *conditionTestChecker) Check(_ context.Context, _ logr.Logger, _ druidv1alpha1.Etcd) condition.Result { - return t.result -} - -func createConditionCheck(conType druidv1alpha1.ConditionType, status druidv1alpha1.ConditionStatus, reason, message string) condition.Checker { - return &conditionTestChecker{ - result: &conditionResult{ - ConType: conType, - ConStatus: status, - ConReason: reason, - ConMessage: message, - }, - } -} - -type etcdMemberResult struct { - id *string - name string - role *druidv1alpha1.EtcdRole - status druidv1alpha1.EtcdMemberConditionStatus - reason string -} - -func (r *etcdMemberResult) ID() *string { - return r.id -} - -func (r *etcdMemberResult) Name() string { - return r.name -} - -func (r *etcdMemberResult) Role() *druidv1alpha1.EtcdRole { - return r.role -} - -func (r *etcdMemberResult) Status() druidv1alpha1.EtcdMemberConditionStatus { - return r.status -} - -func (r *etcdMemberResult) Reason() string { - return r.reason -} - -type etcdMemberTestChecker struct { - results []etcdMemberResult -} - -func (t *etcdMemberTestChecker) Check(_ context.Context, _ druidv1alpha1.Etcd) []etcdmember.Result { - var results []etcdmember.Result - for _, r := range t.results { - result := r - results = append(results, &result) - } - - return results -} - -func createEtcdMemberCheck(results ...etcdMemberResult) etcdmember.Checker { - return &etcdMemberTestChecker{ - results: results, - } -} diff --git a/pkg/health/status/status_suite_test.go b/pkg/health/status/status_suite_test.go deleted file mode 100644 index 290e28fa2..000000000 --- a/pkg/health/status/status_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package status_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestStatus(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Status Suite") -} diff --git a/pkg/mapper/etcd_to_secret.go b/pkg/mapper/etcd_to_secret.go deleted file mode 100644 index 4bf797dd7..000000000 --- a/pkg/mapper/etcd_to_secret.go +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/go-logr/logr" - - "github.com/gardener/gardener/pkg/controllerutils/mapper" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -type etcdToSecretMapper struct{} - -func (m *etcdToSecretMapper) Map(_ context.Context, _ logr.Logger, _ client.Reader, obj client.Object) []reconcile.Request { - etcd, ok := obj.(*druidv1alpha1.Etcd) - if !ok { - return nil - } - - var requests []reconcile.Request - - if etcd.Spec.Etcd.ClientUrlTLS != nil { - requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - }}) - - requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - }}) - - requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - }}) - } - - if etcd.Spec.Etcd.PeerUrlTLS != nil { - requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name, - }}) - - requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name, - }}) - } - - if etcd.Spec.Backup.Store != nil && etcd.Spec.Backup.Store.SecretRef != nil { - requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Spec.Backup.Store.SecretRef.Name, - }}) - } - - return requests -} - -// EtcdToSecret returns a mapper that returns a request for the Secret resources -// referenced in the Etcd for which an event happened. -func EtcdToSecret() mapper.Mapper { - return &etcdToSecretMapper{} -} diff --git a/pkg/mapper/etcd_to_secret_test.go b/pkg/mapper/etcd_to_secret_test.go deleted file mode 100644 index ed7526817..000000000 --- a/pkg/mapper/etcd_to_secret_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper_test - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - . "github.com/gardener/etcd-druid/pkg/mapper" - "github.com/go-logr/logr" - - "github.com/gardener/gardener/pkg/controllerutils/mapper" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -var _ = Describe("EtcdToSecret", func() { - var ( - ctx = context.Background() - m mapper.Mapper - etcd *druidv1alpha1.Etcd - logger logr.Logger - - namespace = "some-namespace" - ) - - BeforeEach(func() { - m = EtcdToSecret() - - etcd = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - }, - } - logger = log.Log.WithName("Test") - }) - - It("should return empty list because Etcd is not referencing secrets", func() { - Expect(m.Map(ctx, logger, nil, etcd)).To(BeEmpty()) - }) - - It("should return four requests because Etcd is referencing secrets", func() { - var ( - secretClientCATLS = "client-url-ca-etcd" - secretClientServerTLS = "client-url-etcd-server-tls" - secretClientClientTLS = "client-url-etcd-client-tls" - secretPeerCATLS = "peer-url-ca-etcd" - secretPeerServerTLS = "peer-url-etcd-server-tls" - secretBackupStore = "backup-store" - ) - - etcd.Spec.Etcd.ClientUrlTLS = &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{Name: secretClientCATLS}, - }, - ServerTLSSecretRef: corev1.SecretReference{Name: secretClientServerTLS}, - ClientTLSSecretRef: corev1.SecretReference{Name: secretClientClientTLS}, - } - - etcd.Spec.Etcd.PeerUrlTLS = &druidv1alpha1.TLSConfig{ - TLSCASecretRef: druidv1alpha1.SecretReference{ - SecretReference: corev1.SecretReference{Name: secretPeerCATLS}, - }, - ServerTLSSecretRef: corev1.SecretReference{Name: secretPeerServerTLS}, - } - - etcd.Spec.Backup.Store = &druidv1alpha1.StoreSpec{ - SecretRef: &corev1.SecretReference{Name: secretBackupStore}, - } - - Expect(m.Map(ctx, logger, nil, etcd)).To(ConsistOf( - reconcile.Request{NamespacedName: types.NamespacedName{ - Name: secretClientCATLS, - Namespace: namespace, - }}, - reconcile.Request{NamespacedName: types.NamespacedName{ - Name: secretClientServerTLS, - Namespace: namespace, - }}, - reconcile.Request{NamespacedName: types.NamespacedName{ - Name: secretClientClientTLS, - Namespace: namespace, - }}, - reconcile.Request{NamespacedName: types.NamespacedName{ - Name: secretPeerCATLS, - Namespace: namespace, - }}, - reconcile.Request{NamespacedName: types.NamespacedName{ - Name: secretPeerServerTLS, - Namespace: namespace, - }}, - reconcile.Request{NamespacedName: types.NamespacedName{ - Name: secretBackupStore, - Namespace: namespace, - }}, - )) - }) -}) diff --git a/pkg/mapper/mapper_suite_test.go b/pkg/mapper/mapper_suite_test.go deleted file mode 100644 index ed8100265..000000000 --- a/pkg/mapper/mapper_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestMapper(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Mapper Suite") -} diff --git a/pkg/mapper/statefulset_to_etcd.go b/pkg/mapper/statefulset_to_etcd.go deleted file mode 100644 index a174a39f7..000000000 --- a/pkg/mapper/statefulset_to_etcd.go +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/gardener/pkg/controllerutils/mapper" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/gardener/etcd-druid/pkg/common" -) - -type statefulSetToEtcdMapper struct { - ctx context.Context - cl client.Client -} - -func (m *statefulSetToEtcdMapper) Map(_ context.Context, _ logr.Logger, _ client.Reader, obj client.Object) []reconcile.Request { - sts, ok := obj.(*appsv1.StatefulSet) - if !ok { - return nil - } - - ownerKey, ok := sts.Annotations[common.GardenerOwnedBy] - if !ok { - return nil - } - - name, namespace, err := cache.SplitMetaNamespaceKey(ownerKey) - if err != nil { - return nil - } - - etcd := &druidv1alpha1.Etcd{} - if err := m.cl.Get(m.ctx, kutil.Key(name, namespace), etcd); err != nil { - return nil - } - - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: etcd.Namespace, - Name: etcd.Name, - }, - }, - } -} - -// StatefulSetToEtcd returns a mapper that returns a request for the Etcd resource -// that owns the StatefulSet for which an event happened. -func StatefulSetToEtcd(ctx context.Context, cl client.Client) mapper.Mapper { - return &statefulSetToEtcdMapper{ - ctx: ctx, - cl: cl, - } -} diff --git a/pkg/mapper/statefulset_to_etcd_test.go b/pkg/mapper/statefulset_to_etcd_test.go deleted file mode 100644 index cce09cc35..000000000 --- a/pkg/mapper/statefulset_to_etcd_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package mapper_test - -import ( - "context" - "fmt" - - "github.com/gardener/gardener/pkg/controllerutils/mapper" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.uber.org/mock/gomock" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - . "github.com/gardener/etcd-druid/pkg/mapper" - mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" -) - -var _ = Describe("Druid Mapper", func() { - var ( - ctx = context.Background() - ctrl *gomock.Controller - c *mockclient.MockClient - logger logr.Logger - - name, namespace, key string - statefulset *appsv1.StatefulSet - mapper mapper.Mapper - ) - - BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) - c = mockclient.NewMockClient(ctrl) - logger = log.Log.WithName("Test") - - name = "etcd-test" - namespace = "test" - key = fmt.Sprintf("%s/%s", namespace, name) - statefulset = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - mapper = StatefulSetToEtcd(ctx, c) - }) - - AfterEach(func() { - ctrl.Finish() - }) - - Describe("#StatefulSetToEtcd", func() { - It("should find related Etcd object", func() { - etcd := &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - - c.EXPECT().Get(ctx, kutil.Key(namespace, name), gomock.AssignableToTypeOf(etcd), gomock.Any()).DoAndReturn( - func(_ context.Context, _ client.ObjectKey, obj *druidv1alpha1.Etcd, _ ...client.GetOption) error { - *obj = *etcd - return nil - }, - ) - - kutil.SetMetaDataAnnotation(statefulset, common.GardenerOwnedBy, key) - - etcds := mapper.Map(ctx, logger, nil, statefulset) - - Expect(etcds).To(ConsistOf( - reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: namespace, - Name: name, - }, - }, - )) - }) - - It("should not find related Etcd object because an error occurred during retrieval", func() { - c.EXPECT().Get(ctx, kutil.Key(namespace, name), gomock.AssignableToTypeOf(&druidv1alpha1.Etcd{})).Return(fmt.Errorf("foo error")) - - kutil.SetMetaDataAnnotation(statefulset, common.GardenerOwnedBy, key) - - etcds := mapper.Map(ctx, logger, nil, statefulset) - - Expect(etcds).To(BeEmpty()) - }) - - It("should not find related Etcd object because owner annotation is not present", func() { - etcds := mapper.Map(ctx, logger, nil, statefulset) - - Expect(etcds).To(BeEmpty()) - }) - - It("should not find related Etcd object because map is called with wrong object", func() { - etcds := mapper.Map(ctx, logger, nil, nil) - - Expect(etcds).To(BeEmpty()) - }) - }) -}) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go deleted file mode 100644 index 762ee8c00..000000000 --- a/pkg/metrics/metrics.go +++ /dev/null @@ -1,110 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package metrics - -import "sort" - -const ( - // LabelSucceeded is a metric label indicating whether associated metric - // series is for success or failure. - LabelSucceeded = "succeeded" - // ValueSucceededTrue is value True for metric label succeeded. - ValueSucceededTrue = "true" - // ValueSucceededFalse is value False for metric label failed. - ValueSucceededFalse = "false" - - // EtcdNamespace is the label for prometheus metrics to indicate etcd namespace - EtcdNamespace = "etcd_namespace" -) - -var ( - // DruidLabels are the labels for prometheus metrics - DruidLabels = map[string][]string{ - LabelSucceeded: { - ValueSucceededFalse, - ValueSucceededTrue, - }, - } -) - -// GenerateLabelCombinations generates combinations of label values for metrics -func GenerateLabelCombinations(labelValues map[string][]string) []map[string]string { - labels := make([]string, len(labelValues)) - valuesList := make([][]string, len(labelValues)) - valueCounts := make([]int, len(labelValues)) - i := 0 - for label := range labelValues { - labels[i] = label - i++ - } - sort.Strings(labels) - for i, label := range labels { - values := make([]string, len(labelValues[label])) - copy(values, labelValues[label]) - valuesList[i] = values - valueCounts[i] = len(values) - } - combinations := getCombinations(valuesList) - - output := make([]map[string]string, len(combinations)) - for i, combination := range combinations { - labelVals := make(map[string]string, len(labels)) - for j := 0; j < len(labels); j++ { - labelVals[labels[j]] = combination[j] - } - output[i] = labelVals - } - return output -} - -// getCombinations returns combinations of slice of string slices -func getCombinations(valuesList [][]string) [][]string { - if len(valuesList) == 0 { - return [][]string{} - } else if len(valuesList) == 1 { - return wrapInSlice(valuesList[0]) - } - - return cartesianProduct(wrapInSlice(valuesList[0]), getCombinations(valuesList[1:])) -} - -// cartesianProduct combines two slices of slice of strings while also -// combining the sub-slices of strings into a single string -// Ex: -// a => [[p,q],[r,s]] -// b => [[1,2],[3,4]] -// Output => [[p,q,1,2],[p,q,3,4],[r,s,1,2],[r,s,3,4]] -func cartesianProduct(a [][]string, b [][]string) [][]string { - output := make([][]string, len(a)*len(b)) - for i := 0; i < len(a); i++ { - for j := 0; j < len(b); j++ { - arr := make([]string, len(a[i])+len(b[j])) - ctr := 0 - for ii := 0; ii < len(a[i]); ii++ { - arr[ctr] = a[i][ii] - ctr++ - } - for jj := 0; jj < len(b[j]); jj++ { - arr[ctr] = b[j][jj] - ctr++ - } - output[(i*len(b))+j] = arr - } - } - return output -} - -// wrapInSlice is a helper function to wrap a slice of strings within -// a slice of slices of strings -// Ex: [p,q,r] -> [[p],[q],[r]] -func wrapInSlice(s []string) [][]string { - output := make([][]string, len(s)) - for i := 0; i < len(output); i++ { - elem := make([]string, 1) - elem[0] = s[i] - output[i] = elem - } - return output -} diff --git a/pkg/metrics/metrics_suite_test.go b/pkg/metrics/metrics_suite_test.go deleted file mode 100644 index 8bf8a48cd..000000000 --- a/pkg/metrics/metrics_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package metrics - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestMetrics(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Metrics Suite") -} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go deleted file mode 100644 index 7250cc7c8..000000000 --- a/pkg/metrics/metrics_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package metrics - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Metrics", func() { - Describe("Testing helper functions for metrics initialization", func() { - Context("Testing wrapInSlice with input", func() { - It("should return expected output", func() { - input := []string{"a", "b", "c"} - expectedOutput := [][]string{{"a"}, {"b"}, {"c"}} - output := wrapInSlice(input) - Expect(output).Should(Equal(expectedOutput)) - }) - }) - Context("Testing cartesianProduct with inputs", func() { - It("should return expected output", func() { - input1 := [][]string{{"p", "q"}, {"r", "s"}} - input2 := [][]string{{"1", "2"}, {"3", "4"}} - expectedOutput := [][]string{ - {"p", "q", "1", "2"}, - {"p", "q", "3", "4"}, - {"r", "s", "1", "2"}, - {"r", "s", "3", "4"}, - } - output := cartesianProduct(input1, input2) - Expect(output).Should(Equal(expectedOutput)) - }) - }) - Context("Testing generateLabelCombinations with input of one label", func() { - It("should return expected output", func() { - input := map[string][]string{ - "a": { - "1", - "2", - "3", - }, - } - expectedOutput := []map[string]string{ - {"a": "1"}, - {"a": "2"}, - {"a": "3"}, - } - output := GenerateLabelCombinations(input) - Expect(output).Should(Equal(expectedOutput)) - }) - }) - Context("Testing generateLabelCombinations with input of two labels", func() { - It("should return expected output", func() { - input := map[string][]string{ - "a": { - "1", - "2", - "3", - }, - "b": { - "4", - "5", - }, - } - expectedOutput := []map[string]string{ - {"a": "1", "b": "4"}, - {"a": "1", "b": "5"}, - {"a": "2", "b": "4"}, - {"a": "2", "b": "5"}, - {"a": "3", "b": "4"}, - {"a": "3", "b": "5"}, - } - output := GenerateLabelCombinations(input) - Expect(output).Should(Equal(expectedOutput)) - }) - }) - }) -}) diff --git a/pkg/mock/controller-runtime/client/doc.go b/pkg/mock/controller-runtime/client/doc.go deleted file mode 100644 index c8b9bca5f..000000000 --- a/pkg/mock/controller-runtime/client/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 -//go:generate mockgen -package client -destination=mocks.go sigs.k8s.io/controller-runtime/pkg/client Client,StatusWriter,Reader,Writer - -package client diff --git a/pkg/mock/controller-runtime/client/mocks.go b/pkg/mock/controller-runtime/client/mocks.go deleted file mode 100644 index a4077a7bb..000000000 --- a/pkg/mock/controller-runtime/client/mocks.go +++ /dev/null @@ -1,518 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: sigs.k8s.io/controller-runtime/pkg/client (interfaces: Client,StatusWriter,Reader,Writer) - -// Package client is a generated GoMock package. -package client - -import ( - context "context" - reflect "reflect" - - gomock "go.uber.org/mock/gomock" - meta "k8s.io/apimachinery/pkg/api/meta" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - client "sigs.k8s.io/controller-runtime/pkg/client" -) - -// MockClient is a mock of Client interface. -type MockClient struct { - ctrl *gomock.Controller - recorder *MockClientMockRecorder -} - -// MockClientMockRecorder is the mock recorder for MockClient. -type MockClientMockRecorder struct { - mock *MockClient -} - -// NewMockClient creates a new mock instance. -func NewMockClient(ctrl *gomock.Controller) *MockClient { - mock := &MockClient{ctrl: ctrl} - mock.recorder = &MockClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClient) EXPECT() *MockClientMockRecorder { - return m.recorder -} - -// Create mocks base method. -func (m *MockClient) Create(arg0 context.Context, arg1 client.Object, arg2 ...client.CreateOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Create", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockClientMockRecorder) Create(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockClient)(nil).Create), varargs...) -} - -// Delete mocks base method. -func (m *MockClient) Delete(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Delete", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockClientMockRecorder) Delete(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockClient)(nil).Delete), varargs...) -} - -// DeleteAllOf mocks base method. -func (m *MockClient) DeleteAllOf(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteAllOfOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "DeleteAllOf", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAllOf indicates an expected call of DeleteAllOf. -func (mr *MockClientMockRecorder) DeleteAllOf(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockClient)(nil).DeleteAllOf), varargs...) -} - -// Get mocks base method. -func (m *MockClient) Get(arg0 context.Context, arg1 types.NamespacedName, arg2 client.Object, arg3 ...client.GetOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Get", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Get indicates an expected call of Get. -func (mr *MockClientMockRecorder) Get(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), varargs...) -} - -// GroupVersionKindFor mocks base method. -func (m *MockClient) GroupVersionKindFor(arg0 runtime.Object) (schema.GroupVersionKind, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GroupVersionKindFor", arg0) - ret0, _ := ret[0].(schema.GroupVersionKind) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GroupVersionKindFor indicates an expected call of GroupVersionKindFor. -func (mr *MockClientMockRecorder) GroupVersionKindFor(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GroupVersionKindFor", reflect.TypeOf((*MockClient)(nil).GroupVersionKindFor), arg0) -} - -// IsObjectNamespaced mocks base method. -func (m *MockClient) IsObjectNamespaced(arg0 runtime.Object) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsObjectNamespaced", arg0) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsObjectNamespaced indicates an expected call of IsObjectNamespaced. -func (mr *MockClientMockRecorder) IsObjectNamespaced(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsObjectNamespaced", reflect.TypeOf((*MockClient)(nil).IsObjectNamespaced), arg0) -} - -// List mocks base method. -func (m *MockClient) List(arg0 context.Context, arg1 client.ObjectList, arg2 ...client.ListOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "List", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// List indicates an expected call of List. -func (mr *MockClientMockRecorder) List(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockClient)(nil).List), varargs...) -} - -// Patch mocks base method. -func (m *MockClient) Patch(arg0 context.Context, arg1 client.Object, arg2 client.Patch, arg3 ...client.PatchOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Patch", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Patch indicates an expected call of Patch. -func (mr *MockClientMockRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockClient)(nil).Patch), varargs...) -} - -// RESTMapper mocks base method. -func (m *MockClient) RESTMapper() meta.RESTMapper { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RESTMapper") - ret0, _ := ret[0].(meta.RESTMapper) - return ret0 -} - -// RESTMapper indicates an expected call of RESTMapper. -func (mr *MockClientMockRecorder) RESTMapper() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RESTMapper", reflect.TypeOf((*MockClient)(nil).RESTMapper)) -} - -// Scheme mocks base method. -func (m *MockClient) Scheme() *runtime.Scheme { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Scheme") - ret0, _ := ret[0].(*runtime.Scheme) - return ret0 -} - -// Scheme indicates an expected call of Scheme. -func (mr *MockClientMockRecorder) Scheme() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Scheme", reflect.TypeOf((*MockClient)(nil).Scheme)) -} - -// Status mocks base method. -func (m *MockClient) Status() client.SubResourceWriter { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Status") - ret0, _ := ret[0].(client.SubResourceWriter) - return ret0 -} - -// Status indicates an expected call of Status. -func (mr *MockClientMockRecorder) Status() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockClient)(nil).Status)) -} - -// SubResource mocks base method. -func (m *MockClient) SubResource(arg0 string) client.SubResourceClient { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SubResource", arg0) - ret0, _ := ret[0].(client.SubResourceClient) - return ret0 -} - -// SubResource indicates an expected call of SubResource. -func (mr *MockClientMockRecorder) SubResource(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubResource", reflect.TypeOf((*MockClient)(nil).SubResource), arg0) -} - -// Update mocks base method. -func (m *MockClient) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.UpdateOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Update", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Update indicates an expected call of Update. -func (mr *MockClientMockRecorder) Update(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockClient)(nil).Update), varargs...) -} - -// MockStatusWriter is a mock of StatusWriter interface. -type MockStatusWriter struct { - ctrl *gomock.Controller - recorder *MockStatusWriterMockRecorder -} - -// MockStatusWriterMockRecorder is the mock recorder for MockStatusWriter. -type MockStatusWriterMockRecorder struct { - mock *MockStatusWriter -} - -// NewMockStatusWriter creates a new mock instance. -func NewMockStatusWriter(ctrl *gomock.Controller) *MockStatusWriter { - mock := &MockStatusWriter{ctrl: ctrl} - mock.recorder = &MockStatusWriterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockStatusWriter) EXPECT() *MockStatusWriterMockRecorder { - return m.recorder -} - -// Create mocks base method. -func (m *MockStatusWriter) Create(arg0 context.Context, arg1, arg2 client.Object, arg3 ...client.SubResourceCreateOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Create", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockStatusWriterMockRecorder) Create(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockStatusWriter)(nil).Create), varargs...) -} - -// Patch mocks base method. -func (m *MockStatusWriter) Patch(arg0 context.Context, arg1 client.Object, arg2 client.Patch, arg3 ...client.SubResourcePatchOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Patch", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Patch indicates an expected call of Patch. -func (mr *MockStatusWriterMockRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockStatusWriter)(nil).Patch), varargs...) -} - -// Update mocks base method. -func (m *MockStatusWriter) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.SubResourceUpdateOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Update", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Update indicates an expected call of Update. -func (mr *MockStatusWriterMockRecorder) Update(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockStatusWriter)(nil).Update), varargs...) -} - -// MockReader is a mock of Reader interface. -type MockReader struct { - ctrl *gomock.Controller - recorder *MockReaderMockRecorder -} - -// MockReaderMockRecorder is the mock recorder for MockReader. -type MockReaderMockRecorder struct { - mock *MockReader -} - -// NewMockReader creates a new mock instance. -func NewMockReader(ctrl *gomock.Controller) *MockReader { - mock := &MockReader{ctrl: ctrl} - mock.recorder = &MockReaderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockReader) EXPECT() *MockReaderMockRecorder { - return m.recorder -} - -// Get mocks base method. -func (m *MockReader) Get(arg0 context.Context, arg1 types.NamespacedName, arg2 client.Object, arg3 ...client.GetOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Get", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Get indicates an expected call of Get. -func (mr *MockReaderMockRecorder) Get(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockReader)(nil).Get), varargs...) -} - -// List mocks base method. -func (m *MockReader) List(arg0 context.Context, arg1 client.ObjectList, arg2 ...client.ListOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "List", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// List indicates an expected call of List. -func (mr *MockReaderMockRecorder) List(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockReader)(nil).List), varargs...) -} - -// MockWriter is a mock of Writer interface. -type MockWriter struct { - ctrl *gomock.Controller - recorder *MockWriterMockRecorder -} - -// MockWriterMockRecorder is the mock recorder for MockWriter. -type MockWriterMockRecorder struct { - mock *MockWriter -} - -// NewMockWriter creates a new mock instance. -func NewMockWriter(ctrl *gomock.Controller) *MockWriter { - mock := &MockWriter{ctrl: ctrl} - mock.recorder = &MockWriterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockWriter) EXPECT() *MockWriterMockRecorder { - return m.recorder -} - -// Create mocks base method. -func (m *MockWriter) Create(arg0 context.Context, arg1 client.Object, arg2 ...client.CreateOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Create", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockWriterMockRecorder) Create(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockWriter)(nil).Create), varargs...) -} - -// Delete mocks base method. -func (m *MockWriter) Delete(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Delete", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockWriterMockRecorder) Delete(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockWriter)(nil).Delete), varargs...) -} - -// DeleteAllOf mocks base method. -func (m *MockWriter) DeleteAllOf(arg0 context.Context, arg1 client.Object, arg2 ...client.DeleteAllOfOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "DeleteAllOf", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAllOf indicates an expected call of DeleteAllOf. -func (mr *MockWriterMockRecorder) DeleteAllOf(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockWriter)(nil).DeleteAllOf), varargs...) -} - -// Patch mocks base method. -func (m *MockWriter) Patch(arg0 context.Context, arg1 client.Object, arg2 client.Patch, arg3 ...client.PatchOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Patch", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Patch indicates an expected call of Patch. -func (mr *MockWriterMockRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockWriter)(nil).Patch), varargs...) -} - -// Update mocks base method. -func (m *MockWriter) Update(arg0 context.Context, arg1 client.Object, arg2 ...client.UpdateOption) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Update", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// Update indicates an expected call of Update. -func (mr *MockWriterMockRecorder) Update(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockWriter)(nil).Update), varargs...) -} diff --git a/pkg/utils/envvar.go b/pkg/utils/envvar.go deleted file mode 100644 index 0e5fecac1..000000000 --- a/pkg/utils/envvar.go +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" -) - -// GetEnvVarFromValue returns environment variable object with the provided name and value -func GetEnvVarFromValue(name, value string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - Value: value, - } -} - -// GetEnvVarFromFieldPath returns environment variable object with provided name and value from field path -func GetEnvVarFromFieldPath(name, fieldPath string) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: fieldPath, - }, - }, - } -} - -// GetEnvVarFromSecret returns environment variable object with provided name and optional value from secret -func GetEnvVarFromSecret(name, secretName, secretKey string, optional bool) corev1.EnvVar { - return corev1.EnvVar{ - Name: name, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, - }, - Key: secretKey, - Optional: pointer.Bool(optional), - }, - }, - } -} - -// GetProviderEnvVars returns provider-specific environment variables for the given store -func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { - var envVars []corev1.EnvVar - - provider, err := StorageProviderFromInfraProvider(store.Provider) - if err != nil { - return nil, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") - } - - const credentialsMountPath = "/var/etcd-backup" - switch provider { - case S3: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, credentialsMountPath)) - - case ABS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, credentialsMountPath)) - - case GCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, "/var/.gcp/serviceaccount.json")) - envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) - - case Swift: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, credentialsMountPath)) - - case OSS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, credentialsMountPath)) - - case ECS: - if store.SecretRef == nil { - return nil, fmt.Errorf("no secretRef could be configured for backup store of ECS") - } - envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSEndpoint, store.SecretRef.Name, "endpoint", false)) - envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSAccessKeyID, store.SecretRef.Name, "accessKeyID", false)) - envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) - - case OCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, credentialsMountPath)) - } - - return envVars, nil -} - -// GetBackupRestoreContainerEnvVars returns backup-restore container environment variables for the given store -func GetBackupRestoreContainerEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { - var envVars []corev1.EnvVar - - envVars = append(envVars, GetEnvVarFromFieldPath(common.EnvPodName, "metadata.name")) - envVars = append(envVars, GetEnvVarFromFieldPath(common.EnvPodNamespace, "metadata.namespace")) - - if store == nil { - return envVars, nil - } - - storageContainer := pointer.StringDeref(store.Container, "") - envVars = append(envVars, GetEnvVarFromValue(common.EnvStorageContainer, storageContainer)) - - providerEnvVars, err := GetProviderEnvVars(store) - if err != nil { - return nil, err - } - envVars = append(envVars, providerEnvVars...) - - return envVars, nil -} diff --git a/pkg/utils/image.go b/pkg/utils/image.go deleted file mode 100755 index 61d785bef..000000000 --- a/pkg/utils/image.go +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/gardener/pkg/utils/imagevector" - "k8s.io/utils/pointer" -) - -func getEtcdImageKeys(useEtcdWrapper bool) (etcdImageKey string, etcdbrImageKey string, alpine string) { - alpine = common.Alpine - switch useEtcdWrapper { - case true: - etcdImageKey = common.EtcdWrapper - etcdbrImageKey = common.BackupRestoreDistroless - default: - etcdImageKey = common.Etcd - etcdbrImageKey = common.BackupRestore - } - return -} - -// GetEtcdImages returns images for etcd and backup-restore by inspecting the etcd spec and the image vector -// and returns the image for the init container by inspecting the image vector. -// It will give preference to images that are set in the etcd spec and only if the image is not found in it should -// it be picked up from the image vector if it's set there. -// A return value of nil for either of the images indicates that the image is not set. -func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcdWrapper bool) (*string, *string, *string, error) { - etcdImageKey, etcdbrImageKey, initContainerImageKey := getEtcdImageKeys(useEtcdWrapper) - etcdImage, err := chooseImage(etcdImageKey, etcd.Spec.Etcd.Image, iv) - if err != nil { - return nil, nil, nil, err - } - etcdBackupRestoreImage, err := chooseImage(etcdbrImageKey, etcd.Spec.Backup.Image, iv) - if err != nil { - return nil, nil, nil, err - } - initContainerImage, err := chooseImage(initContainerImageKey, nil, iv) - if err != nil { - return nil, nil, nil, err - } - - return etcdImage, etcdBackupRestoreImage, initContainerImage, nil -} - -// chooseImage selects an image based on the given key, specImage, and image vector. -// It returns the specImage if it is not nil; otherwise, it searches for the image in the image vector. -func chooseImage(key string, specImage *string, iv imagevector.ImageVector) (*string, error) { - if specImage != nil { - return specImage, nil - } - // Check if this image is present in the image vector. - ivImage, err := imagevector.FindImages(iv, []string{key}) - if err != nil { - return nil, err - } - return pointer.String(ivImage[key].String()), nil -} - -// GetEtcdBackupRestoreImage returns the image for backup-restore from the given image vector. -func GetEtcdBackupRestoreImage(iv imagevector.ImageVector, useEtcdWrapper bool) (*string, error) { - _, etcdbrImageKey, _ := getEtcdImageKeys(useEtcdWrapper) - return chooseImage(etcdbrImageKey, nil, iv) -} - -// GetInitContainerImage returns the image for init container from the given image vector. -func GetInitContainerImage(iv imagevector.ImageVector) (*string, error) { - return chooseImage(common.Alpine, nil, iv) -} diff --git a/pkg/utils/image_test.go b/pkg/utils/image_test.go deleted file mode 100644 index e0f61a336..000000000 --- a/pkg/utils/image_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - testutils "github.com/gardener/etcd-druid/test/utils" - - "github.com/gardener/gardener/pkg/utils/imagevector" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/utils/pointer" -) - -var _ = Describe("Image retrieval tests", func() { - - const ( - etcdName = "etcd-test-0" - namespace = "default" - ) - var ( - imageVector imagevector.ImageVector - etcd *druidv1alpha1.Etcd - err error - ) - - It("etcd spec defines etcd and backup-restore images", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() - imageVector = createImageVector(true, true, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - Expect(etcdImage).To(Equal(etcd.Spec.Etcd.Image)) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) - - It("etcd spec has no image defined and image vector has both images set", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Etcd.Image = nil - etcd.Spec.Backup.Image = nil - imageVector = createImageVector(true, true, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - vectorEtcdImage, err := imageVector.FindImage(common.Etcd) - Expect(err).To(BeNil()) - Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestore) - Expect(err).To(BeNil()) - Expect(*etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) - - It("etcd spec only has backup-restore image and image-vector has only etcd image", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Etcd.Image = nil - imageVector = createImageVector(true, false, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - vectorEtcdImage, err := imageVector.FindImage(common.Etcd) - Expect(err).To(BeNil()) - Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - Expect(etcdBackupRestoreImage).To(Equal(etcd.Spec.Backup.Image)) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) - - It("both spec and image vector do not have backup-restore image", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Backup.Image = nil - imageVector = createImageVector(true, false, false, false) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, false) - Expect(err).ToNot(BeNil()) - Expect(etcdImage).To(BeNil()) - Expect(etcdBackupRestoreImage).To(BeNil()) - Expect(initContainerImage).To(BeNil()) - }) - - It("etcd spec has no images defined, image vector has all images, and UseEtcdWrapper feature gate is turned on", func() { - etcd = testutils.EtcdBuilderWithDefaults(etcdName, namespace).Build() - Expect(err).To(BeNil()) - etcd.Spec.Etcd.Image = nil - etcd.Spec.Backup.Image = nil - imageVector = createImageVector(true, true, true, true) - etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, imageVector, true) - Expect(err).To(BeNil()) - Expect(etcdImage).ToNot(BeNil()) - vectorEtcdImage, err := imageVector.FindImage(common.EtcdWrapper) - Expect(err).To(BeNil()) - Expect(*etcdImage).To(Equal(vectorEtcdImage.String())) - Expect(etcdBackupRestoreImage).ToNot(BeNil()) - vectorBackupRestoreImage, err := imageVector.FindImage(common.BackupRestoreDistroless) - Expect(err).To(BeNil()) - Expect(*etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) - vectorInitContainerImage, err := imageVector.FindImage(common.Alpine) - Expect(err).To(BeNil()) - Expect(*initContainerImage).To(Equal(vectorInitContainerImage.String())) - }) -}) - -func createImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperImage, withBackupRestoreDistrolessImage bool) imagevector.ImageVector { - var imageSources []*imagevector.ImageSource - const ( - repo = "test-repo" - etcdTag = "etcd-test-tag" - etcdWrapperTag = "etcd-wrapper-test-tag" - backupRestoreTag = "backup-restore-test-tag" - backupRestoreDistrolessTag = "backup-restore-distroless-test-tag" - initContainerTag = "init-container-test-tag" - ) - if withEtcdImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.Etcd, - Repository: repo, - Tag: pointer.String(etcdTag), - }) - } - if withBackupRestoreImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.BackupRestore, - Repository: repo, - Tag: pointer.String(backupRestoreTag), - }) - - } - if withEtcdWrapperImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.EtcdWrapper, - Repository: repo, - Tag: pointer.String(etcdWrapperTag), - }) - } - if withBackupRestoreDistrolessImage { - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.BackupRestoreDistroless, - Repository: repo, - Tag: pointer.String(backupRestoreDistrolessTag), - }) - } - imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.Alpine, - Repository: repo, - Tag: pointer.String(initContainerTag), - }) - return imageSources -} diff --git a/pkg/utils/lease.go b/pkg/utils/lease.go deleted file mode 100644 index 81831d7fb..000000000 --- a/pkg/utils/lease.go +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "context" - "strconv" - - "github.com/gardener/etcd-druid/pkg/common" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/go-logr/logr" - coordinationv1 "k8s.io/api/coordination/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// IsPeerURLTLSEnabled checks if the TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. -func IsPeerURLTLSEnabled(ctx context.Context, cli client.Client, namespace, etcdName string, logger logr.Logger) (bool, error) { - var tlsEnabledValues []bool - labels := GetMemberLeaseLabels(etcdName) - leaseList := &coordinationv1.LeaseList{} - if err := cli.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { - return false, err - } - for _, lease := range leaseList.Items { - tlsEnabled := parseAndGetTLSEnabledValue(lease, logger) - if tlsEnabled != nil { - tlsEnabledValues = append(tlsEnabledValues, *tlsEnabled) - } - } - tlsEnabled := true - for _, v := range tlsEnabledValues { - tlsEnabled = tlsEnabled && v - } - return tlsEnabled, nil -} - -// PurposeMemberLease is a constant used as a purpose for etcd member lease objects. -const PurposeMemberLease = "etcd-member-lease" - -// GetMemberLeaseLabels creates a map of default labels for member lease. -func GetMemberLeaseLabels(etcdName string) map[string]string { - return map[string]string{ - common.GardenerOwnedBy: etcdName, - v1beta1constants.GardenerPurpose: PurposeMemberLease, - } -} - -func parseAndGetTLSEnabledValue(lease coordinationv1.Lease, logger logr.Logger) *bool { - const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" - if lease.Annotations != nil { - if tlsEnabledStr, ok := lease.Annotations[peerURLTLSEnabledKey]; ok { - tlsEnabled, err := strconv.ParseBool(tlsEnabledStr) - if err != nil { - logger.Error(err, "tls-enabled value is not a valid boolean", "namespace", lease.Namespace, "leaseName", lease.Name) - return nil - } - return &tlsEnabled - } - logger.V(4).Info("tls-enabled annotation not present for lease.", "namespace", lease.Namespace, "leaseName", lease.Name) - } - return nil -} diff --git a/pkg/utils/miscellaneous.go b/pkg/utils/miscellaneous.go deleted file mode 100644 index 60b24b168..000000000 --- a/pkg/utils/miscellaneous.go +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "fmt" - "time" - - "github.com/robfig/cron/v3" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// MergeStringMaps merges the content of the newMaps with the oldMap. If a key already exists then -// it gets overwritten by the last value with the same key. -func MergeStringMaps(oldMap map[string]string, newMaps ...map[string]string) map[string]string { - var out map[string]string - - if oldMap != nil { - out = make(map[string]string) - } - for k, v := range oldMap { - out[k] = v - } - - for _, newMap := range newMaps { - if newMap != nil && out == nil { - out = make(map[string]string) - } - - for k, v := range newMap { - out[k] = v - } - } - - return out -} - -func nameAndNamespace(namespaceOrName string, nameOpt ...string) (namespace, name string) { - if len(nameOpt) > 1 { - panic(fmt.Sprintf("more than name/namespace for key specified: %s/%v", namespaceOrName, nameOpt)) - } - if len(nameOpt) == 0 { - name = namespaceOrName - return - } - namespace = namespaceOrName - name = nameOpt[0] - return -} - -// Key creates a new client.ObjectKey from the given parameters. -// There are only two ways to call this function: -// - If only namespaceOrName is set, then a client.ObjectKey with name set to namespaceOrName is returned. -// - If namespaceOrName and one nameOpt is given, then a client.ObjectKey with namespace set to namespaceOrName -// and name set to nameOpt[0] is returned. -// -// For all other cases, this method panics. -func Key(namespaceOrName string, nameOpt ...string) client.ObjectKey { - namespace, name := nameAndNamespace(namespaceOrName, nameOpt...) - return client.ObjectKey{Namespace: namespace, Name: name} -} - -// Max returns the larger of x or y. -func Max(x, y int) int { - if y > x { - return y - } - return x -} - -// ComputeScheduleInterval computes the interval between two activations for the given cron schedule. -// Assumes that every cron activation is at equal intervals apart, based on cron schedules such as -// "once every X hours", "once every Y days", "at 1:00pm on every Tuesday", etc. -// TODO: write a new function to accurately compute the previous activation time from the cron schedule -// in order to compute when the previous activation of the cron schedule was supposed to have occurred, -// instead of relying on the assumption that all the cron activations are evenly spaced. -func ComputeScheduleInterval(cronSchedule string) (time.Duration, error) { - schedule, err := cron.ParseStandard(cronSchedule) - if err != nil { - return 0, err - } - - nextScheduledTime := schedule.Next(time.Now()) - nextNextScheduledTime := schedule.Next(nextScheduledTime) - return nextNextScheduledTime.Sub(nextScheduledTime), nil -} diff --git a/pkg/utils/miscellaneous_test.go b/pkg/utils/miscellaneous_test.go deleted file mode 100644 index 43d5a6bb5..000000000 --- a/pkg/utils/miscellaneous_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Tests for miscellaneous utility functions", func() { - Describe("#ComputeScheduleInterval", func() { - It("should compute schedule duration as 24h when schedule set to 00:00 everyday", func() { - schedule := "0 0 * * *" - duration, err := ComputeScheduleInterval(schedule) - Expect(err).To(Not(HaveOccurred())) - Expect(duration).To(Equal(24 * time.Hour)) - }) - - It("should compute schedule duration as 24h when schedule set to 15:30 everyday", func() { - schedule := "30 15 * * *" - duration, err := ComputeScheduleInterval(schedule) - Expect(err).To(Not(HaveOccurred())) - Expect(duration).To(Equal(24 * time.Hour)) - }) - - It("should compute schedule duration as 12h when schedule set to 03:30 and 15:30 everyday", func() { - schedule := "30 3,15 * * *" - duration, err := ComputeScheduleInterval(schedule) - Expect(err).To(Not(HaveOccurred())) - Expect(duration).To(Equal(12 * time.Hour)) - }) - - It("should compute schedule duration as 7d when schedule set to 15:30 on every Tuesday", func() { - schedule := "30 15 * * 2" - duration, err := ComputeScheduleInterval(schedule) - Expect(err).To(Not(HaveOccurred())) - Expect(duration).To(Equal(7 * 24 * time.Hour)) - }) - }) -}) diff --git a/pkg/utils/statefulset.go b/pkg/utils/statefulset.go deleted file mode 100644 index 09c204c42..000000000 --- a/pkg/utils/statefulset.go +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "context" - "fmt" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - appsv1 "k8s.io/api/apps/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// IsStatefulSetReady checks whether the given StatefulSet is ready and up-to-date. -// A StatefulSet is considered healthy if its controller observed its current revision, -// it is not in an update (i.e. UpdateRevision is empty) and if its current replicas are equal to -// desired replicas specified in ETCD specs. -// It returns ready status (bool) and in case it is not ready then the second return value holds the reason. -func IsStatefulSetReady(etcdReplicas int32, statefulSet *appsv1.StatefulSet) (bool, string) { - if statefulSet.Status.ObservedGeneration < statefulSet.Generation { - return false, fmt.Sprintf("observed generation %d is outdated in comparison to generation %d", statefulSet.Status.ObservedGeneration, statefulSet.Generation) - } - if statefulSet.Status.ReadyReplicas < etcdReplicas { - return false, fmt.Sprintf("not enough ready replicas (%d/%d)", statefulSet.Status.ReadyReplicas, etcdReplicas) - } - if statefulSet.Status.CurrentRevision != statefulSet.Status.UpdateRevision { - return false, fmt.Sprintf("Current StatefulSet revision %s is older than the updated StatefulSet revision %s)", statefulSet.Status.CurrentRevision, statefulSet.Status.UpdateRevision) - } - if statefulSet.Status.CurrentReplicas != statefulSet.Status.UpdatedReplicas { - return false, fmt.Sprintf("StatefulSet status.CurrentReplicas (%d) != status.UpdatedReplicas (%d)", statefulSet.Status.CurrentReplicas, statefulSet.Status.UpdatedReplicas) - } - return true, "" -} - -// GetStatefulSet fetches StatefulSet created for the etcd. -func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { - sts := &appsv1.StatefulSet{} - - if err := cl.Get(ctx, client.ObjectKey{ - Name: etcd.Name, - Namespace: etcd.Namespace, - }, sts); err != nil { - - if apierrors.IsNotFound(err) { - return nil, nil - } - return nil, err - } - - return sts, nil -} diff --git a/pkg/utils/statefulset_test.go b/pkg/utils/statefulset_test.go deleted file mode 100644 index a7df321e5..000000000 --- a/pkg/utils/statefulset_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - testutils "github.com/gardener/etcd-druid/test/utils" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("tests for statefulset utility functions", func() { - - var ( - ctx context.Context - fakeClient = fake.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - ) - - Describe("#IsStatefulSetReady", func() { - const ( - stsName = "etcd-test-0" - stsNamespace = "test-ns" - ) - It("statefulset has less number of ready replicas as compared to configured etcd replicas", func() { - sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 2) - sts.Generation = 1 - sts.Status.ObservedGeneration = 1 - sts.Status.Replicas = 2 - sts.Status.ReadyReplicas = 2 - ready, reasonMsg := IsStatefulSetReady(3, sts) - Expect(ready).To(BeFalse()) - Expect(reasonMsg).ToNot(BeNil()) - }) - - It("statefulset has equal number of replicas as defined in etcd but observed generation is outdated", func() { - sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) - sts.Generation = 2 - sts.Status.ObservedGeneration = 1 - sts.Status.Replicas = 3 - sts.Status.ReadyReplicas = 3 - ready, reasonMsg := IsStatefulSetReady(3, sts) - Expect(ready).To(BeFalse()) - Expect(reasonMsg).ToNot(BeNil()) - }) - - It("statefulset has equal number of replicas as defined in etcd and observed generation = generation", func() { - sts := testutils.CreateStatefulSet(stsName, stsNamespace, uuid.NewUUID(), 3) - testutils.SetStatefulSetReady(sts) - ready, reasonMsg := IsStatefulSetReady(3, sts) - Expect(ready).To(BeTrue()) - Expect(len(reasonMsg)).To(BeZero()) - }) - }) - - Describe("#GetStatefulSet", func() { - var ( - etcd *druidv1alpha1.Etcd - stsListToCleanup *appsv1.StatefulSetList - ) - const ( - testEtcdName = "etcd-test-0" - testNamespace = "test-ns" - ) - - BeforeEach(func() { - ctx = context.TODO() - stsListToCleanup = &appsv1.StatefulSetList{} - etcd = testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() - }) - - AfterEach(func() { - for _, sts := range stsListToCleanup.Items { - Expect(fakeClient.Delete(ctx, &sts)).To(Succeed()) - } - }) - - It("no statefulset is found irrespective of ownership", func() { - sts, err := GetStatefulSet(ctx, fakeClient, etcd) - Expect(err).To(BeNil()) - Expect(sts).To(BeNil()) - }) - - It("statefulset is present but it is not owned by etcd", func() { - sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, uuid.NewUUID(), 3) - Expect(fakeClient.Create(ctx, sts)).To(Succeed()) - stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) - foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) - Expect(err).To(BeNil()) - Expect(foundSts).To(BeNil()) - }) - - It("found statefulset owned by etcd", func() { - sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, 3) - Expect(fakeClient.Create(ctx, sts)).To(Succeed()) - stsListToCleanup.Items = append(stsListToCleanup.Items, *sts) - foundSts, err := GetStatefulSet(ctx, fakeClient, etcd) - Expect(err).To(BeNil()) - Expect(foundSts).ToNot(BeNil()) - Expect(foundSts.UID).To(Equal(sts.UID)) - }) - }) -}) diff --git a/pkg/utils/store.go b/pkg/utils/store.go deleted file mode 100644 index fc1eb7695..000000000 --- a/pkg/utils/store.go +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "context" - "fmt" - "strings" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // LocalProviderDefaultMountPath is the default path where the buckets directory is mounted. - LocalProviderDefaultMountPath = "/etc/gardener/local-backupbuckets" - // EtcdBackupSecretHostPath is the hostPath field in the etcd-backup secret. - EtcdBackupSecretHostPath = "hostPath" -) - -const ( - aws = "aws" - azure = "azure" - gcp = "gcp" - alicloud = "alicloud" - openstack = "openstack" - dell = "dell" - openshift = "openshift" -) - -const ( - // S3 is a constant for the AWS and S3 compliant storage provider. - S3 = "S3" - // ABS is a constant for the Azure storage provider. - ABS = "ABS" - // GCS is a constant for the Google storage provider. - GCS = "GCS" - // OSS is a constant for the Alicloud storage provider. - OSS = "OSS" - // Swift is a constant for the OpenStack storage provider. - Swift = "Swift" - // Local is a constant for the Local storage provider. - Local = "Local" - // ECS is a constant for the EMC storage provider. - ECS = "ECS" - // OCS is a constant for the OpenShift storage provider. - OCS = "OCS" -) - -// GetHostMountPathFromSecretRef returns the hostPath configured for the given store. -func GetHostMountPathFromSecretRef(ctx context.Context, client client.Client, logger logr.Logger, store *druidv1alpha1.StoreSpec, namespace string) (string, error) { - if store.SecretRef == nil { - logger.Info("secretRef is not defined for store, using default hostPath", "namespace", namespace) - return LocalProviderDefaultMountPath, nil - } - - secret := &corev1.Secret{} - if err := client.Get(ctx, Key(namespace, store.SecretRef.Name), secret); err != nil { - return "", err - } - - hostPath, ok := secret.Data[EtcdBackupSecretHostPath] - if !ok { - return LocalProviderDefaultMountPath, nil - } - - return string(hostPath), nil -} - -// StorageProviderFromInfraProvider converts infra to object store provider. -func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (string, error) { - if infra == nil || len(*infra) == 0 { - return "", nil - } - - switch *infra { - case aws, S3: - return S3, nil - case azure, ABS: - return ABS, nil - case alicloud, OSS: - return OSS, nil - case openstack, Swift: - return Swift, nil - case gcp, GCS: - return GCS, nil - case dell, ECS: - return ECS, nil - case openshift, OCS: - return OCS, nil - case Local, druidv1alpha1.StorageProvider(strings.ToLower(Local)): - return Local, nil - default: - return "", fmt.Errorf("unsupported storage provider: %v", *infra) - } -} diff --git a/pkg/utils/store_test.go b/pkg/utils/store_test.go deleted file mode 100644 index 14a40c714..000000000 --- a/pkg/utils/store_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "context" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/client/kubernetes" - "github.com/gardener/gardener/pkg/utils/test/matchers" - "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - corev1 "k8s.io/api/core/v1" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Store tests", func() { - - Describe("#GetHostMountPathFromSecretRef", func() { - var ( - ctx context.Context - fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() - storeSpec *druidv1alpha1.StoreSpec - sec *corev1.Secret - logger = logr.Discard() - ) - const ( - testNamespace = "test-ns" - ) - - BeforeEach(func() { - ctx = context.Background() - storeSpec = &druidv1alpha1.StoreSpec{} - }) - - AfterEach(func() { - if sec != nil { - err := fakeClient.Delete(ctx, sec) - if err != nil { - Expect(err).To(matchers.BeNotFoundError()) - } - } - }) - - It("no secret ref configured, should return default mount path", func() { - hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(HaveOccurred()) - Expect(hostMountPath).To(Equal(LocalProviderDefaultMountPath)) - }) - - It("secret ref points to an unknown secret, should return an error", func() { - storeSpec.SecretRef = &corev1.SecretReference{ - Name: "not-to-be-found-secret-ref", - Namespace: testNamespace, - } - _, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(BeNil()) - Expect(err).To(matchers.BeNotFoundError()) - }) - - It("secret ref points to a secret whose data does not have path set, should return default mount path", func() { - const secretName = "backup-secret" - sec = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: testNamespace, - }, - Data: map[string][]byte{"bucketName": []byte("NDQ5YjEwZj")}, - } - Expect(fakeClient.Create(ctx, sec)).To(Succeed()) - storeSpec.SecretRef = &corev1.SecretReference{ - Name: "backup-secret", - Namespace: testNamespace, - } - hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(HaveOccurred()) - Expect(hostMountPath).To(Equal(LocalProviderDefaultMountPath)) - }) - - It("secret ref points to a secret whose data has a path, should return the path defined in secret.Data", func() { - const ( - secretName = "backup-secret" - hostPath = "/var/data/etcd-backup" - ) - sec = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: testNamespace, - }, - Data: map[string][]byte{"bucketName": []byte("NDQ5YjEwZj"), "hostPath": []byte(hostPath)}, - } - Expect(fakeClient.Create(ctx, sec)).To(Succeed()) - storeSpec.SecretRef = &corev1.SecretReference{ - Name: "backup-secret", - Namespace: testNamespace, - } - hostMountPath, err := GetHostMountPathFromSecretRef(ctx, fakeClient, logger, storeSpec, testNamespace) - Expect(err).ToNot(HaveOccurred()) - Expect(hostMountPath).To(Equal(hostPath)) - }) - }) -}) diff --git a/pkg/utils/utils_suite_test.go b/pkg/utils/utils_suite_test.go deleted file mode 100644 index 7faf50c41..000000000 --- a/pkg/utils/utils_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestUtils(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Utils Suite") -} diff --git a/pkg/version/version.go b/pkg/version/version.go deleted file mode 100644 index 641a6b146..000000000 --- a/pkg/version/version.go +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package version - -var ( - // These variables are typically populated using -ldflags settings when building the binary - - // Version stores the etcd-druid binary version. - Version string - // GitSHA stores the etcd-druid binary code commit SHA on git. - GitSHA string -) diff --git a/test/e2e/etcd_multi_node_test.go b/test/e2e/etcd_multi_node_test.go index 532ba3493..09a2455d6 100644 --- a/test/e2e/etcd_multi_node_test.go +++ b/test/e2e/etcd_multi_node_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" diff --git a/test/e2e/utils.go b/test/e2e/utils.go index bbbbb9f55..7adb3f97b 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -15,7 +15,7 @@ import ( "time" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/etcd-backup-restore/pkg/snapstore" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" diff --git a/test/integration/controllers/assets/assets.go b/test/integration/controllers/assets/assets.go index 5599144ed..0105535dc 100644 --- a/test/integration/controllers/assets/assets.go +++ b/test/integration/controllers/assets/assets.go @@ -7,7 +7,7 @@ package assets import ( "path/filepath" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/gomega" diff --git a/test/integration/controllers/compaction/compaction_suite_test.go b/test/integration/controllers/compaction/compaction_suite_test.go index 890b01292..68dffae21 100644 --- a/test/integration/controllers/compaction/compaction_suite_test.go +++ b/test/integration/controllers/compaction/compaction_suite_test.go @@ -10,7 +10,7 @@ import ( "github.com/gardener/etcd-druid/test/integration/controllers/assets" - "github.com/gardener/etcd-druid/controllers/compaction" + "github.com/gardener/etcd-druid/internal/controller/compaction" "github.com/gardener/etcd-druid/test/integration/setup" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index ad7ed76e0..f0679a142 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -10,8 +10,8 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" diff --git a/test/integration/controllers/custodian/custodian_suite_test.go b/test/integration/controllers/custodian/custodian_suite_test.go deleted file mode 100644 index 482df5492..000000000 --- a/test/integration/controllers/custodian/custodian_suite_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package custodian - -import ( - "context" - "testing" - "time" - - "github.com/gardener/etcd-druid/controllers/custodian" - "github.com/gardener/etcd-druid/test/integration/controllers/assets" - "github.com/gardener/etcd-druid/test/integration/setup" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -var ( - k8sClient client.Client - intTestEnv *setup.IntegrationTestEnv - namespace string -) - -const ( - testNamespacePrefix = "custodian-" -) - -func TestCustodianController(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs( - t, - "Custodian Controller Suite", - ) -} - -var _ = BeforeSuite(func() { - crdPaths := []string{assets.GetEtcdCrdPath()} - ctx, cancelFn := context.WithTimeout(context.Background(), time.Minute) - defer cancelFn() - intTestEnv = setup.NewIntegrationTestEnv(testNamespacePrefix, "custodian-int-tests", crdPaths) - intTestEnv.RegisterReconcilers(func(mgr manager.Manager) { - reconciler := custodian.NewReconciler(mgr, &custodian.Config{ - Workers: 5, - SyncPeriod: 15 * time.Second, - EtcdMember: custodian.EtcdMemberConfig{ - NotReadyThreshold: 1 * time.Minute, - UnknownThreshold: 2 * time.Minute, - }, - }) - Expect(reconciler.RegisterWithManager(ctx, mgr, true)).To(Succeed()) - }).StartManager() - k8sClient = intTestEnv.K8sClient - namespace = intTestEnv.TestNs.Name -}) diff --git a/test/integration/controllers/custodian/reconciler_test.go b/test/integration/controllers/custodian/reconciler_test.go deleted file mode 100644 index fb1f4ab1a..000000000 --- a/test/integration/controllers/custodian/reconciler_test.go +++ /dev/null @@ -1,191 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package custodian - -import ( - "context" - "fmt" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - componentpdb "github.com/gardener/etcd-druid/pkg/component/etcd/poddisruptionbudget" - testutils "github.com/gardener/etcd-druid/test/utils" - - "github.com/gardener/gardener/pkg/utils/test/matchers" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - timeout = time.Minute * 5 - pollingInterval = time.Second * 2 -) - -var _ = Describe("Custodian Controller", func() { - Describe("Updating Etcd status", func() { - Context("when statefulset status is updated", func() { - var ( - instance *druidv1alpha1.Etcd - sts *appsv1.StatefulSet - ctx = context.TODO() - name = "foo11" - ) - - BeforeEach(func() { - instance = testutils.EtcdBuilderWithDefaults(name, namespace).Build() - - Expect(k8sClient.Create(ctx, instance)).To(Succeed()) - // wait for Etcd creation to succeed - Eventually(func() error { - return k8sClient.Get(ctx, types.NamespacedName{ - Name: name, - Namespace: namespace, - }, instance) - }, timeout, pollingInterval).Should(BeNil()) - - // update etcd status.ObservedGeneration to metadata.Generation so that custodian predicate is satisfied - instance.Status.ObservedGeneration = pointer.Int64(instance.Generation) - Expect(k8sClient.Status().Update(ctx, instance)).To(Succeed()) - // wait for etcd status update to succeed - Eventually(func() error { - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), instance); err != nil { - return err - } - if *instance.Status.ObservedGeneration != instance.Generation { - return fmt.Errorf("etcd not reconciled yet") - } - return nil - }, timeout, pollingInterval).Should(Succeed()) - - // create sts manually, since there is no running etcd controller to create sts upon Etcd creation - sts = testutils.CreateStatefulSet(name, namespace, instance.UID, instance.Spec.Replicas) - Expect(k8sClient.Create(ctx, sts)).To(Succeed()) - // wait for sts creation to succeed - Eventually(func() error { - return k8sClient.Get(ctx, types.NamespacedName{ - Name: name, - Namespace: namespace, - }, sts) - }, timeout, pollingInterval).Should(BeNil()) - }) - - It("should update value of Etcd.Status.ReadyReplicas to value of Statefulset.Status.ReadyReplicas", func() { - sts.Status.Replicas = 1 - sts.Status.ReadyReplicas = 1 - sts.Status.ObservedGeneration = 2 - Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) - - // wait for sts status update to succeed - Eventually(func() error { - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { - return err - } - if sts.Status.ReadyReplicas != 1 { - return fmt.Errorf("statefulset ReadyReplicas should be equal to 1") - } - return nil - }, timeout, pollingInterval).Should(Succeed()) - - // Wait for desired ReadyReplicas value to be reflected in etcd status - Eventually(func() error { - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), instance); err != nil { - return err - } - - if int(instance.Status.ReadyReplicas) != 1 { - return fmt.Errorf("etcd ready replicas should be equal to 1") - } - return nil - }, timeout, pollingInterval).Should(BeNil()) - }) - - It("should mark statefulset status not ready when no ready replicas in statefulset", func() { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), sts) - Expect(err).ToNot(HaveOccurred()) - - // Forcefully change ReadyReplicas in statefulset to 0 - sts.Status.ReadyReplicas = 0 - Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) - - // wait for sts status update to succeed - Eventually(func() error { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), sts) - if err != nil { - return err - } - - if sts.Status.ReadyReplicas > 0 { - return fmt.Errorf("no readyreplicas of statefulset should exist at this point") - } - - return nil - }, timeout, pollingInterval).Should(BeNil()) - - // wait for etcd status to reflect the change in ReadyReplicas - Eventually(func() error { - err = k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), instance) - if err != nil { - return err - } - - if instance.Status.ReadyReplicas > 0 { - return fmt.Errorf("ReadyReplicas should be zero in ETCD instance") - } - - return nil - }, timeout, pollingInterval).Should(BeNil()) - }) - - AfterEach(func() { - // Delete etcd instance - Expect(k8sClient.Delete(ctx, instance)).To(Succeed()) - // Delete sts - Expect(k8sClient.Delete(ctx, sts)).To(Succeed()) - - // Wait for etcd to be deleted - Eventually(func() error { - return k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), instance) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - - // Wait for sts to be deleted - Eventually(func() error { - return k8sClient.Get(ctx, client.ObjectKeyFromObject(sts), sts) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - }) - }) - }) - - Describe("PodDisruptionBudget", func() { - Context("minAvailable of PodDisruptionBudget", func() { - When("having a single node cluster", func() { - etcd := testutils.EtcdBuilderWithDefaults("test", "default").WithReadyStatus().Build() - - Expect(len(etcd.Status.Members)).To(BeEquivalentTo(1)) - - It("should be set to 0", func() { - etcd.Spec.Replicas = 1 - Expect(componentpdb.CalculatePDBMinAvailable(etcd)).To(BeEquivalentTo(0)) - etcd.Spec.Replicas = 0 - Expect(componentpdb.CalculatePDBMinAvailable(etcd)).To(BeEquivalentTo(0)) - }) - }) - - When("having a multi node cluster", func() { - etcd := testutils.EtcdBuilderWithDefaults("test", "default").WithReplicas(3).WithReadyStatus().Build() - - Expect(len(etcd.Status.Members)).To(BeEquivalentTo(3)) - - It("should calculate the value correctly", func() { - Expect(componentpdb.CalculatePDBMinAvailable(etcd)).To(BeEquivalentTo(2)) - }) - }) - }) - }) - -}) diff --git a/test/integration/controllers/etcd/etcd_suite_test.go b/test/integration/controllers/etcd/etcd_suite_test.go deleted file mode 100644 index cdbecec08..000000000 --- a/test/integration/controllers/etcd/etcd_suite_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcd - -import ( - "testing" - - "github.com/gardener/etcd-druid/controllers/etcd" - "github.com/gardener/etcd-druid/pkg/features" - "github.com/gardener/etcd-druid/test/integration/controllers/assets" - "github.com/gardener/etcd-druid/test/integration/setup" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/client-go/rest" - "k8s.io/component-base/featuregate" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -var ( - intTestEnv *setup.IntegrationTestEnv - k8sClient client.Client - restConfig *rest.Config - namespace string -) - -const ( - testNamespacePrefix = "etcd-" -) - -func TestEtcdController(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs( - t, - "Etcd Controller Suite", - ) -} - -var _ = BeforeSuite(func() { - crdPaths := []string{assets.GetEtcdCrdPath()} - imageVector := assets.CreateImageVector() - - intTestEnv = setup.NewIntegrationTestEnv(testNamespacePrefix, "compaction-int-tests", crdPaths) - intTestEnv.RegisterReconcilers(func(mgr manager.Manager) { - reconciler, err := etcd.NewReconcilerWithImageVector(mgr, &etcd.Config{ - Workers: 5, - DisableEtcdServiceAccountAutomount: false, - FeatureGates: map[featuregate.Feature]bool{ - features.UseEtcdWrapper: true, - }, - }, imageVector) - Expect(err).To(BeNil()) - Expect(reconciler.RegisterWithManager(mgr, true)).To(Succeed()) - }).StartManager() - k8sClient = intTestEnv.K8sClient - restConfig = intTestEnv.RestConfig - namespace = intTestEnv.TestNs.Name -}) diff --git a/test/integration/controllers/etcd/reconciler_test.go b/test/integration/controllers/etcd/reconciler_test.go deleted file mode 100644 index c21777a0c..000000000 --- a/test/integration/controllers/etcd/reconciler_test.go +++ /dev/null @@ -1,1649 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package etcd - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "time" - - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/controllers/etcd" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" - "github.com/gardener/etcd-druid/test/integration/controllers/assets" - testutils "github.com/gardener/etcd-druid/test/utils" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - gardenerUtils "github.com/gardener/gardener/pkg/utils" - "github.com/gardener/gardener/pkg/utils/imagevector" - "github.com/gardener/gardener/pkg/utils/test/matchers" - "github.com/ghodss/yaml" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - timeout = time.Minute * 5 - pollingInterval = time.Second * 2 - etcdConfig = "etcd.conf.yaml" - backupRestore = "backup-restore" - metricsKey = "metrics" -) - -var ( - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - defaultStorageCapacity = resource.MustParse("16Gi") - deltaSnapShotMemLimit = resource.MustParse("100Mi") - autoCompactionMode = druidv1alpha1.Periodic - autoCompactionRetention = "2m" - quota = resource.MustParse("8Gi") - maxBackups = 7 - imageNames = []string{ - common.Etcd, - common.BackupRestore, - common.EtcdWrapper, - common.BackupRestoreDistroless, - } -) - -var _ = Describe("Etcd Controller", func() { - //Reconciliation of new etcd resource deployment without any existing statefulsets. - Context("when adding etcd resources", func() { - var ( - err error - instance *druidv1alpha1.Etcd - sts *appsv1.StatefulSet - svc *corev1.Service - ctx = context.TODO() - ) - - BeforeEach(func() { - instance = testutils.EtcdBuilderWithDefaults("foo1", namespace).Build() - - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(errs).Should(BeNil()) - Expect(k8sClient.Create(context.TODO(), instance)).To(Succeed()) - - sts = &appsv1.StatefulSet{} - // Wait until StatefulSet has been created by controller - Eventually(func() error { - return k8sClient.Get(context.TODO(), types.NamespacedName{ - Name: instance.Name, - Namespace: instance.Namespace, - }, sts) - }, timeout, pollingInterval).Should(BeNil()) - - svc = &corev1.Service{} - // Wait until Service has been created by controller - Eventually(func() error { - return k8sClient.Get(context.TODO(), types.NamespacedName{ - Name: instance.GetClientServiceName(), - Namespace: instance.Namespace, - }, svc) - }, timeout, pollingInterval).Should(BeNil()) - - }) - It("should create and adopt statefulset", func() { - ctx := context.TODO() - - testutils.SetStatefulSetReady(sts) - err = k8sClient.Status().Update(ctx, sts) - Eventually(func() (bool, error) { return testutils.IsStatefulSetCorrectlyReconciled(ctx, k8sClient, instance, sts) }, timeout, pollingInterval).Should(BeTrue()) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() (*bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), instance); err != nil { - return nil, err - } - return instance.Status.Ready, nil - }, timeout, pollingInterval).Should(Equal(pointer.Bool(true))) - }) - It("should create and adopt statefulset and printing events", func() { - // Check StatefulSet requirements - Expect(len(sts.Spec.VolumeClaimTemplates)).To(Equal(1)) - Expect(sts.Spec.Replicas).To(PointTo(Equal(int32(1)))) - - // Create PVC - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s-%d", sts.Spec.VolumeClaimTemplates[0].Name, sts.Name, 0), - Namespace: sts.Namespace, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - } - Expect(k8sClient.Create(context.TODO(), pvc)).To(Succeed()) - - // Create PVC warning Event - pvcMessage := "Failed to provision volume" - Expect(k8sClient.Create(context.TODO(), &corev1.Event{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-event-1", - Namespace: pvc.Namespace, - }, - InvolvedObject: corev1.ObjectReference{ - APIVersion: "v1", - Kind: "PersistentVolumeClaim", - Name: pvc.Name, - Namespace: pvc.Namespace, - }, - Type: corev1.EventTypeWarning, - Message: pvcMessage, - })).To(Succeed()) - - // Eventually, warning message should be reflected in `etcd` object status. - Eventually(func() string { - if err := k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance); err != nil { - return "" - } - if instance.Status.LastError == nil { - return "" - } - return *instance.Status.LastError - }, timeout, pollingInterval).Should(ContainSubstring(pvcMessage)) - }) - AfterEach(func() { - // Delete `etcd` instance - Expect(k8sClient.Delete(context.TODO(), instance)).To(Succeed()) - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), &druidv1alpha1.Etcd{}) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - // Delete service manually because garbage collection is not available in `envtest` - Expect(k8sClient.Delete(context.TODO(), svc)).To(Succeed()) - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(svc), &corev1.Service{}) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - - }) - }) - - DescribeTable("when etcd resource is created", - func(etcdName string, withTLS bool, provider druidv1alpha1.StorageProvider, etcdWithDefaults bool, validate func(*druidv1alpha1.Etcd, *appsv1.StatefulSet, *corev1.ConfigMap, *corev1.Service, *corev1.Service)) { - var ( - err error - s *appsv1.StatefulSet - cm *corev1.ConfigMap - clSvc, prSvc *corev1.Service - sa *corev1.ServiceAccount - role *rbac.Role - rb *rbac.RoleBinding - ctx = context.TODO() - instance *druidv1alpha1.Etcd - instanceBuilder *testutils.EtcdBuilder - ) - if etcdWithDefaults { - instanceBuilder = testutils.EtcdBuilderWithDefaults(etcdName, namespace) - } else { - instanceBuilder = testutils.EtcdBuilderWithoutDefaults(etcdName, namespace) - } - if withTLS { - instanceBuilder.WithPeerTLS().WithClientTLS() - } - instance = instanceBuilder.WithStorageProvider(provider).Build() - - if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { - By("create backup-store secrets") - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(errs).Should(BeNil()) - } - - By("create etcd instance and check if it has been created") - err = k8sClient.Create(context.TODO(), instance) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() error { - return k8sClient.Get(ctx, client.ObjectKeyFromObject(instance), instance) - }).Should(Not(HaveOccurred())) - - By("check if statefulset has reconciled") - s = &appsv1.StatefulSet{} - Eventually(func() (bool, error) { return testutils.IsStatefulSetCorrectlyReconciled(ctx, k8sClient, instance, s) }, timeout, pollingInterval).Should(BeTrue()) - - By("check if configmap has reconciled") - cm = &corev1.ConfigMap{} - Eventually(func() error { return testutils.ConfigMapIsCorrectlyReconciled(k8sClient, timeout, instance, cm) }, timeout, pollingInterval).Should(BeNil()) - - By("check if client service has reconciled") - clSvc = &corev1.Service{} - Eventually(func() error { return testutils.ClientServiceIsCorrectlyReconciled(k8sClient, timeout, instance, clSvc) }, timeout, pollingInterval).Should(BeNil()) - - By("check if peer service has reconciled") - prSvc = &corev1.Service{} - Eventually(func() error { return testutils.PeerServiceIsCorrectlyReconciled(k8sClient, timeout, instance, prSvc) }, timeout, pollingInterval).Should(BeNil()) - - By("check if service account has reconciled") - sa = &corev1.ServiceAccount{} - Eventually(func() error { return testutils.ServiceAccountIsCorrectlyReconciled(k8sClient, timeout, instance, sa) }, timeout, pollingInterval).Should(BeNil()) - - By("check if role has reconciled") - role = &rbac.Role{} - Eventually(func() error { return testutils.RoleIsCorrectlyReconciled(k8sClient, timeout, instance, role) }, timeout, pollingInterval).Should(BeNil()) - - By("check if rolebinding has reconciled") - rb = &rbac.RoleBinding{} - Eventually(func() error { return testutils.RoleBindingIsCorrectlyReconciled(k8sClient, timeout, instance, rb) }, timeout, pollingInterval).Should(BeNil()) - - validate(instance, s, cm, clSvc, prSvc) - validateRole(instance, role) - - testutils.SetStatefulSetReady(s) - err = k8sClient.Status().Update(context.TODO(), s) - Expect(err).NotTo(HaveOccurred()) - }, - Entry("if fields are not set in etcd.Spec, the statefulset should reflect the spec changes", "foo28", false, druidv1alpha1.StorageProvider("Local"), false, validateDefaultValuesForEtcd), - Entry("if fields are set in etcd.Spec and TLS enabled, the resources should reflect the spec changes", "foo29", true, druidv1alpha1.StorageProvider("Local"), true, validateEtcd), - Entry("if the store is GCS, the statefulset should reflect the spec changes", "foo30", true, druidv1alpha1.StorageProvider("gcp"), true, validateStoreGCP), - Entry("if the store is S3, the statefulset should reflect the spec changes", "foo31", true, druidv1alpha1.StorageProvider("aws"), true, validateStoreAWS), - Entry("if the store is ABS, the statefulset should reflect the spec changes", "foo32", true, druidv1alpha1.StorageProvider("azure"), true, validateStoreAzure), - Entry("if the store is Swift, the statefulset should reflect the spec changes", "foo33", true, druidv1alpha1.StorageProvider("openstack"), true, validateStoreOpenstack), - Entry("if the store is OSS, the statefulset should reflect the spec changes", "foo34", true, druidv1alpha1.StorageProvider("alicloud"), true, validateStoreAlicloud), - ) -}) - -var _ = Describe("Multinode ETCD", func() { - //Reconciliation of new etcd resource deployment without any existing statefulsets. - Context("when adding etcd resources", func() { - var ( - instance *druidv1alpha1.Etcd - sts *appsv1.StatefulSet - svc *corev1.Service - ctx = context.TODO() - ) - - BeforeEach(func() { - instance = testutils.EtcdBuilderWithDefaults("foo82", namespace).Build() - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(errs).Should(BeNil()) - }) - It("should create the statefulset based on the replicas in ETCD CR", func() { - // First delete existing statefulset if any. - // This is required due to a bug - sts = &appsv1.StatefulSet{} - sts.Name = instance.Name - sts.Namespace = instance.Namespace - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, sts))).To(Succeed()) - - By("update replicas in ETCD resource with 0") - instance.Spec.Replicas = 4 - Expect(k8sClient.Create(ctx, instance)).To(Succeed()) - - Eventually(func() error { - return k8sClient.Get(ctx, types.NamespacedName{ - Name: instance.Name, - Namespace: instance.Namespace, - }, instance) - }, timeout, pollingInterval).Should(BeNil()) - - By("no StatefulSet has been created by controller as even number of replicas are not allowed") - Eventually(func() error { - return k8sClient.Get(ctx, types.NamespacedName{ - Name: instance.Name, - Namespace: instance.Namespace, - }, sts) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - - By("update replicas in ETCD resource with 3") - patch := client.MergeFrom(instance.DeepCopy()) - instance.Spec.Replicas = 3 - Expect(k8sClient.Patch(ctx, instance, patch)).To(Succeed()) - - By("statefulsets are created when ETCD replicas are odd number") - Eventually(func() (bool, error) { return testutils.IsStatefulSetCorrectlyReconciled(ctx, k8sClient, instance, sts) }, timeout, pollingInterval).Should(BeTrue()) - Expect(int(*sts.Spec.Replicas)).To(Equal(3)) - - By("client Service has been created by controller") - svc = &corev1.Service{} - Eventually(func() error { return testutils.ClientServiceIsCorrectlyReconciled(k8sClient, timeout, instance, svc) }, timeout, pollingInterval).Should(BeNil()) - - By("should raise an event if annotation to ignore reconciliation is applied on ETCD CR") - patch = client.MergeFrom(instance.DeepCopy()) - annotations := utils.MergeStringMaps( - map[string]string{ - etcd.IgnoreReconciliationAnnotation: "true", - }, - instance.Annotations, - ) - instance.Annotations = annotations - Expect(k8sClient.Patch(ctx, instance, patch)).To(Succeed()) - - clientset, _ := kubernetes.NewForConfig(restConfig) - Eventually(func() error { - events, err := clientset.CoreV1().Events(instance.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - fmt.Printf("The error is : %v", err) - return err - } - - if events == nil || len(events.Items) == 0 { - return fmt.Errorf("no events generated for annotation to ignore reconciliation") - } - - for _, event := range events.Items { - if event.Reason == "ReconciliationIgnored" { - return nil - } - } - return nil - - }, timeout, pollingInterval).Should(BeNil()) - - By("delete `etcd` instance") - Expect(client.IgnoreNotFound(k8sClient.Delete(context.TODO(), instance))).To(Succeed()) - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(instance), &druidv1alpha1.Etcd{}) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - - By("delete service manually because garbage collection is not available in `envtest`") - if svc != nil { - Expect(k8sClient.Delete(context.TODO(), svc)).To(Succeed()) - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(svc), &corev1.Service{}) - }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) - } - }) - }) - DescribeTable("configmaps are mounted properly when ETCD replicas are odd number", func(etcdName string, replicas int) { - var ( - err error - sts *appsv1.StatefulSet - cm *corev1.ConfigMap - svc *corev1.Service - ctx = context.TODO() - instance *druidv1alpha1.Etcd - ) - instance = testutils.EtcdBuilderWithDefaults(etcdName, namespace).WithReplicas(int32(replicas)).Build() - - if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errs := testutils.CreateSecrets(ctx, k8sClient, instance.Namespace, storeSecret) - Expect(errs).Should(BeNil()) - } - err = k8sClient.Create(context.TODO(), instance) - Expect(err).NotTo(HaveOccurred()) - sts = &appsv1.StatefulSet{} - Eventually(func() (bool, error) { return testutils.IsStatefulSetCorrectlyReconciled(ctx, k8sClient, instance, sts) }, timeout, pollingInterval).Should(BeTrue()) - cm = &corev1.ConfigMap{} - Eventually(func() error { return testutils.ConfigMapIsCorrectlyReconciled(k8sClient, timeout, instance, cm) }, timeout, pollingInterval).Should(BeNil()) - svc = &corev1.Service{} - Eventually(func() error { return testutils.ClientServiceIsCorrectlyReconciled(k8sClient, timeout, instance, svc) }, timeout, pollingInterval).Should(BeNil()) - - // Validate statefulset - Expect(*sts.Spec.Replicas).To(Equal(instance.Spec.Replicas)) - - if instance.Spec.Replicas == 1 { - matcher := fmt.Sprintf("initial-cluster: foo83-0=http://foo83-0.foo83-peer.%s.svc:2380", namespace) - Expect(strings.Contains(cm.Data["etcd.conf.yaml"], matcher)).To(BeTrue()) - } - - if instance.Spec.Replicas > 1 { - matcher := fmt.Sprintf("initial-cluster: foo84-0=http://foo84-0.foo84-peer.%s.svc:2380,foo84-1=http://foo84-1.foo84-peer.%s.svc:2380,foo84-2=http://foo84-2.foo84-peer.%s.svc:2380", namespace, namespace, namespace) - Expect(strings.Contains(cm.Data["etcd.conf.yaml"], matcher)).To(BeTrue()) - } - }, - Entry("verify configmap mount path and etcd.conf.yaml when replica is 1 ", "foo83", 1), - Entry("verify configmap mount path and etcd.conf.yaml when replica is 3 ", "foo84", 3), - ) -}) - -func validateRole(instance *druidv1alpha1.Etcd, role *rbac.Role) { - Expect(*role).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.GetRoleName()), - "Namespace": Equal(instance.Namespace), - "Labels": MatchKeys(IgnoreExtras, Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - "OwnerReferences": MatchElements(testutils.OwnerRefIterator, IgnoreExtras, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(instance.Name), - "UID": Equal(instance.UID), - "Controller": PointTo(Equal(true)), - "BlockOwnerDeletion": PointTo(Equal(true)), - }), - }), - }), - "Rules": MatchAllElements(testutils.RuleIterator, Elements{ - "coordination.k8s.io": MatchFields(IgnoreExtras, Fields{ - "APIGroups": MatchAllElements(testutils.StringArrayIterator, Elements{ - "coordination.k8s.io": Equal("coordination.k8s.io"), - }), - "Resources": MatchAllElements(testutils.StringArrayIterator, Elements{ - "leases": Equal("leases"), - }), - "Verbs": MatchAllElements(testutils.StringArrayIterator, Elements{ - "list": Equal("list"), - "get": Equal("get"), - "update": Equal("update"), - "patch": Equal("patch"), - "watch": Equal("watch"), - }), - }), - "apps": MatchFields(IgnoreExtras, Fields{ - "APIGroups": MatchAllElements(testutils.StringArrayIterator, Elements{ - "apps": Equal("apps"), - }), - "Resources": MatchAllElements(testutils.StringArrayIterator, Elements{ - "statefulsets": Equal("statefulsets"), - }), - "Verbs": MatchAllElements(testutils.StringArrayIterator, Elements{ - "list": Equal("list"), - "get": Equal("get"), - "update": Equal("update"), - "patch": Equal("patch"), - "watch": Equal("watch"), - }), - }), - "": MatchFields(IgnoreExtras, Fields{ - "APIGroups": MatchAllElements(testutils.StringArrayIterator, Elements{ - "": Equal(""), - }), - "Resources": MatchAllElements(testutils.StringArrayIterator, Elements{ - "pods": Equal("pods"), - }), - "Verbs": MatchAllElements(testutils.StringArrayIterator, Elements{ - "list": Equal("list"), - "get": Equal("get"), - "watch": Equal("watch"), - }), - }), - }), - })) -} - -func validateDefaultValuesForEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { - configYML := cm.Data[etcdConfig] - config := map[string]interface{}{} - err := yaml.Unmarshal([]byte(configYML), &config) - Expect(err).NotTo(HaveOccurred()) - - // Validate ETCD annotation for configmap checksum - jsonString, err := json.Marshal(cm.Data) - Expect(err).NotTo(HaveOccurred()) - configMapChecksum := gardenerUtils.ComputeSHA256Hex(jsonString) - - // Validate Metrics MetricsLevel - Expect(instance.Spec.Etcd.Metrics).To(BeNil()) - Expect(config).To(HaveKeyWithValue(metricsKey, string(druidv1alpha1.Basic))) - - // Validate DefragmentationSchedule *string - Expect(instance.Spec.Etcd.DefragmentationSchedule).To(BeNil()) - - // Validate ServerPort and ClientPort - Expect(instance.Spec.Etcd.ServerPort).To(BeNil()) - Expect(instance.Spec.Etcd.ClientPort).To(BeNil()) - - Expect(instance.Spec.Etcd.Image).To(BeNil()) - imageVector := assets.CreateImageVector() - Expect(err).NotTo(HaveOccurred()) - images, err := imagevector.FindImages(imageVector, imageNames) - Expect(err).NotTo(HaveOccurred()) - - // Validate Resources - // resources: - // requests: - // cpu: 50m - // memory: 128Mi - Expect(instance.Spec.Etcd.Resources).To(BeNil()) - - // Validate TLS. Ensure that enableTLS flag is not triggered in the go-template - Expect(instance.Spec.Etcd.PeerUrlTLS).To(BeNil()) - - Expect(config).To(MatchKeys(IgnoreExtras, Keys{ - "name": Equal(fmt.Sprintf("etcd-%s", instance.UID[:6])), - "data-dir": Equal("/var/etcd/data/new.etcd"), - "metrics": Equal(string(druidv1alpha1.Basic)), - "snapshot-count": Equal(float64(75000)), - "enable-v2": Equal(false), - "quota-backend-bytes": Equal(float64(8589934592)), - "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", clientPort)), - "advertise-client-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "http", prSvc.Name, instance.Namespace, clientPort)), - "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", serverPort)), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "http", prSvc.Name, instance.Namespace, serverPort)), - "initial-cluster-token": Equal("etcd-cluster"), - "initial-cluster-state": Equal("new"), - "auto-compaction-mode": Equal(string(druidv1alpha1.Periodic)), - "auto-compaction-retention": Equal("30m"), - })) - - Expect(*clSvc).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.GetClientServiceName()), - "Namespace": Equal(instance.Namespace), - "Labels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - "OwnerReferences": MatchElements(testutils.OwnerRefIterator, IgnoreExtras, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(instance.Name), - "UID": Equal(instance.UID), - "Controller": PointTo(Equal(true)), - "BlockOwnerDeletion": PointTo(Equal(true)), - }), - }), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "Type": Equal(corev1.ServiceTypeClusterIP), - "SessionAffinity": Equal(corev1.ServiceAffinityNone), - "Selector": MatchKeys(IgnoreExtras, Keys{ - "instance": Equal(instance.Name), - "name": Equal("etcd"), - }), - "Ports": MatchElements(testutils.ServicePortIterator, IgnoreExtras, Elements{ - "client": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client"), - "Protocol": Equal(corev1.ProtocolTCP), - "Port": Equal(clientPort), - "TargetPort": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(clientPort), - }), - }), - "server": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("server"), - "Protocol": Equal(corev1.ProtocolTCP), - "Port": Equal(serverPort), - "TargetPort": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(serverPort), - }), - }), - "backuprestore": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("backuprestore"), - "Protocol": Equal(corev1.ProtocolTCP), - "Port": Equal(backupPort), - "TargetPort": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(backupPort), - }), - }), - }), - }), - })) - - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.Name), - "Namespace": Equal(instance.Namespace), - "Annotations": MatchAllKeys(Keys{ - "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", instance.Namespace, instance.Name)), - "gardener.cloud/owner-type": Equal("etcd"), - "app": Equal("etcd-statefulset"), - "role": Equal("test"), - "instance": Equal(instance.Name), - "checksum/etcd-configmap": Equal(configMapChecksum), - "name": Equal("etcd"), - }), - "Labels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ - "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), - }), - "ServiceName": Equal(instance.GetPeerServiceName()), - "Replicas": PointTo(Equal(instance.Spec.Replicas)), - "Selector": PointTo(MatchFields(IgnoreExtras, Fields{ - "MatchLabels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - })), - "Template": MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "app": Equal("etcd-statefulset"), - "role": Equal("test"), - "instance": Equal(instance.Name), - }), - "Labels": MatchAllKeys(Keys{ - "app": Equal("etcd-statefulset"), - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "HostAliases": MatchAllElements(testutils.HostAliasIterator, Elements{ - "127.0.0.1": MatchFields(IgnoreExtras, Fields{ - "IP": Equal("127.0.0.1"), - "Hostnames": MatchAllElements(testutils.CmdIterator, Elements{ - fmt.Sprintf("%s-local", instance.Name): Equal(fmt.Sprintf("%s-local", instance.Name)), - }), - }), - }), - "PriorityClassName": Equal(""), - "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ - common.Etcd: MatchFields(IgnoreExtras, Fields{ - "Ports": ConsistOf([]corev1.ContainerPort{ - { - Name: "server", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: serverPort, - }, - { - Name: "client", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: clientPort, - }, - }), - "Args": MatchAllElements(testutils.CmdIterator, Elements{ - "start-etcd": Equal("start-etcd"), - "--backup-restore-tls-enabled=false": Equal("--backup-restore-tls-enabled=false"), - fmt.Sprintf("--backup-restore-host-port=%s-local:%d", instance.Name, backupPort): Equal(fmt.Sprintf("--backup-restore-host-port=%s-local:%d", instance.Name, backupPort)), - fmt.Sprintf("--etcd-server-name=%s-local", instance.Name): Equal(fmt.Sprintf("--etcd-server-name=%s-local", instance.Name)), - }), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "Image": Equal(fmt.Sprintf("%s:%s", images[common.EtcdWrapper].Repository, *images[common.EtcdWrapper].Tag)), - "Resources": MatchFields(IgnoreExtras, Fields{ - "Requests": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceCPU: Equal(resource.MustParse("50m")), - corev1.ResourceMemory: Equal(resource.MustParse("128Mi")), - }), - }), - "ReadinessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ - "ProbeHandler": MatchFields(IgnoreExtras, Fields{ - "HTTPGet": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal("/healthz"), - "Port": Equal(intstr.FromInt(int(backupPort))), - "Scheme": Equal(corev1.URISchemeHTTP), - })), - }), - "InitialDelaySeconds": Equal(int32(15)), - "PeriodSeconds": Equal(int32(5)), - "FailureThreshold": Equal(int32(5)), - })), - "VolumeMounts": MatchAllElements(testutils.VolumeMountIterator, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.Name), - "MountPath": Equal("/var/etcd/data/"), - }), - }), - }), - - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchAllElements(testutils.CmdIterator, Elements{ - "server": Equal("server"), - "--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"), - "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp"), - "--insecure-transport=true": Equal("--insecure-transport=true"), - "--insecure-skip-tls-verify=true": Equal("--insecure-skip-tls-verify=true"), - "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), - "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), - "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), - "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), - - fmt.Sprintf("--delta-snapshot-memory-limit=%d", deltaSnapShotMemLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", deltaSnapShotMemLimit.Value())), - fmt.Sprintf("--garbage-collection-policy=%s", druidv1alpha1.GarbageCollectionPolicyLimitBased): Equal(fmt.Sprintf("--garbage-collection-policy=%s", druidv1alpha1.GarbageCollectionPolicyLimitBased)), - fmt.Sprintf("--endpoints=http://%s-local:%d", instance.Name, clientPort): Equal(fmt.Sprintf("--endpoints=http://%s-local:%d", instance.Name, clientPort)), - fmt.Sprintf("--service-endpoints=http://%s:%d", instance.GetClientServiceName(), clientPort): Equal(fmt.Sprintf("--service-endpoints=http://%s:%d", instance.GetClientServiceName(), clientPort)), - fmt.Sprintf("--embedded-etcd-quota-bytes=%d", quota.Value()): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", quota.Value())), - fmt.Sprintf("--max-backups=%d", maxBackups): Equal(fmt.Sprintf("--max-backups=%d", maxBackups)), - fmt.Sprintf("--auto-compaction-mode=%s", druidv1alpha1.Periodic): Equal(fmt.Sprintf("--auto-compaction-mode=%s", druidv1alpha1.Periodic)), - fmt.Sprintf("--auto-compaction-retention=%s", "30m"): Equal(fmt.Sprintf("--auto-compaction-retention=%s", "30m")), - fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", "15m"): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", "15m")), - fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", "15m"): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", "15m")), - }), - "Ports": ConsistOf([]corev1.ContainerPort{ - { - Name: "server", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: backupPort, - }, - }), - "Image": Equal(fmt.Sprintf("%s:%s", images[common.BackupRestoreDistroless].Repository, *images[common.BackupRestoreDistroless].Tag)), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "VolumeMounts": MatchAllElements(testutils.VolumeMountIterator, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.Name), - "MountPath": Equal("/var/etcd/data"), - }), - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "MountPath": Equal("/var/etcd/config/"), - }), - }), - "Env": MatchAllElements(testutils.EnvIterator, Elements{ - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - }), - "SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{ - "Capabilities": PointTo(MatchFields(IgnoreExtras, Fields{ - "Add": ConsistOf([]corev1.Capability{ - "SYS_PTRACE", - }), - })), - })), - }), - }), - "ShareProcessNamespace": Equal(pointer.Bool(true)), - "Volumes": MatchAllElements(testutils.VolumeIterator, Elements{ - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ - "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(fmt.Sprintf("etcd-bootstrap-%s", string(instance.UID[:6]))), - }), - "DefaultMode": PointTo(Equal(int32(0644))), - "Items": MatchAllElements(testutils.KeyIterator, Elements{ - "etcd.conf.yaml": MatchFields(IgnoreExtras, Fields{ - "Key": Equal("etcd.conf.yaml"), - "Path": Equal("etcd.conf.yaml"), - }), - }), - })), - }), - }), - }), - }), - }), - "VolumeClaimTemplates": MatchAllElements(testutils.PVCIterator, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.Name), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "AccessModes": MatchAllElements(testutils.AccessModeIterator, Elements{ - "ReadWriteOnce": Equal(corev1.ReadWriteOnce), - }), - "Resources": MatchFields(IgnoreExtras, Fields{ - "Requests": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceStorage: Equal(defaultStorageCapacity), - }), - }), - }), - }), - }), - }), - })) -} - -func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { - configYML := cm.Data[etcdConfig] - config := map[string]interface{}{} - err := yaml.Unmarshal([]byte(configYML), &config) - Expect(err).NotTo(HaveOccurred()) - - // Validate ETCD annotation for configmap checksum - jsonString, err := json.Marshal(cm.Data) - Expect(err).NotTo(HaveOccurred()) - configMapChecksum := gardenerUtils.ComputeSHA256Hex(jsonString) - - // Validate Metrics MetricsLevel - Expect(instance.Spec.Etcd.Metrics).NotTo(BeNil()) - Expect(config).To(HaveKeyWithValue(metricsKey, string(*instance.Spec.Etcd.Metrics))) - - // Validate DefragmentationSchedule *string - Expect(instance.Spec.Etcd.DefragmentationSchedule).NotTo(BeNil()) - - // Validate Image - Expect(instance.Spec.Etcd.Image).NotTo(BeNil()) - - // Validate Resources - Expect(instance.Spec.Etcd.Resources).NotTo(BeNil()) - - store, err := utils.StorageProviderFromInfraProvider(instance.Spec.Backup.Store.Provider) - Expect(err).NotTo(HaveOccurred()) - - Expect(*cm).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(fmt.Sprintf("etcd-bootstrap-%s", string(instance.UID[:6]))), - "Namespace": Equal(instance.Namespace), - "Labels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - "OwnerReferences": MatchElements(testutils.OwnerRefIterator, IgnoreExtras, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(instance.Name), - "UID": Equal(instance.UID), - "Controller": PointTo(Equal(true)), - "BlockOwnerDeletion": PointTo(Equal(true)), - }), - }), - }), - })) - - Expect(config).To(MatchKeys(IgnoreExtras, Keys{ - "name": Equal(fmt.Sprintf("etcd-%s", instance.UID[:6])), - "data-dir": Equal("/var/etcd/data/new.etcd"), - "metrics": Equal(string(*instance.Spec.Etcd.Metrics)), - "snapshot-count": Equal(float64(75000)), - "enable-v2": Equal(false), - "quota-backend-bytes": Equal(float64(instance.Spec.Etcd.Quota.Value())), - - "client-transport-security": MatchKeys(IgnoreExtras, Keys{ - "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), - "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), - "client-cert-auth": Equal(true), - "trusted-ca-file": Equal("/var/etcd/ssl/client/ca/ca.crt"), - "auto-tls": Equal(false), - }), - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *instance.Spec.Etcd.ClientPort)), - "advertise-client-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", prSvc.Name, instance.Namespace, *instance.Spec.Etcd.ClientPort)), - - "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ - "cert-file": Equal("/var/etcd/ssl/peer/server/tls.crt"), - "key-file": Equal("/var/etcd/ssl/peer/server/tls.key"), - "client-cert-auth": Equal(true), - "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), - "auto-tls": Equal(false), - }), - "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *instance.Spec.Etcd.ServerPort)), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", prSvc.Name, instance.Namespace, *instance.Spec.Etcd.ServerPort)), - - "initial-cluster-token": Equal("etcd-cluster"), - "initial-cluster-state": Equal("new"), - "auto-compaction-mode": Equal(string(*instance.Spec.Common.AutoCompactionMode)), - "auto-compaction-retention": Equal(*instance.Spec.Common.AutoCompactionRetention), - })) - - Expect(*clSvc).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.GetClientServiceName()), - "Namespace": Equal(instance.Namespace), - "Labels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - "OwnerReferences": MatchElements(testutils.OwnerRefIterator, IgnoreExtras, Elements{ - instance.Name: MatchFields(IgnoreExtras, Fields{ - "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), - "Kind": Equal("Etcd"), - "Name": Equal(instance.Name), - "UID": Equal(instance.UID), - "Controller": PointTo(Equal(true)), - "BlockOwnerDeletion": PointTo(Equal(true)), - }), - }), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "Type": Equal(corev1.ServiceTypeClusterIP), - "SessionAffinity": Equal(corev1.ServiceAffinityNone), - "Selector": MatchKeys(IgnoreExtras, Keys{ - "instance": Equal(instance.Name), - "name": Equal("etcd"), - }), - "Ports": MatchElements(testutils.ServicePortIterator, IgnoreExtras, Elements{ - "client": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client"), - "Protocol": Equal(corev1.ProtocolTCP), - "Port": Equal(*instance.Spec.Etcd.ClientPort), - "TargetPort": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(*instance.Spec.Etcd.ClientPort), - }), - }), - "server": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("server"), - "Protocol": Equal(corev1.ProtocolTCP), - "Port": Equal(*instance.Spec.Etcd.ServerPort), - "TargetPort": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(*instance.Spec.Etcd.ServerPort), - }), - }), - "backuprestore": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("backuprestore"), - "Protocol": Equal(corev1.ProtocolTCP), - "Port": Equal(*instance.Spec.Backup.Port), - "TargetPort": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(*instance.Spec.Backup.Port), - }), - }), - }), - }), - })) - - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.Name), - "Namespace": Equal(instance.Namespace), - "Annotations": MatchAllKeys(Keys{ - "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", instance.Namespace, instance.Name)), - "gardener.cloud/owner-type": Equal("etcd"), - "app": Equal("etcd-statefulset"), - "role": Equal("test"), - "instance": Equal(instance.Name), - "checksum/etcd-configmap": Equal(configMapChecksum), - "name": Equal("etcd"), - }), - "Labels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - }), - - "Spec": MatchFields(IgnoreExtras, Fields{ - "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ - "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), - }), - "ServiceName": Equal(instance.GetPeerServiceName()), - "Replicas": PointTo(Equal(instance.Spec.Replicas)), - "Selector": PointTo(MatchFields(IgnoreExtras, Fields{ - "MatchLabels": MatchAllKeys(Keys{ - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - })), - "Template": MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "app": Equal("etcd-statefulset"), - "role": Equal("test"), - "instance": Equal(instance.Name), - }), - "Labels": MatchAllKeys(Keys{ - "app": Equal("etcd-statefulset"), - "name": Equal("etcd"), - "instance": Equal(instance.Name), - }), - }), - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "HostAliases": MatchAllElements(testutils.HostAliasIterator, Elements{ - "127.0.0.1": MatchFields(IgnoreExtras, Fields{ - "IP": Equal("127.0.0.1"), - "Hostnames": MatchAllElements(testutils.CmdIterator, Elements{ - fmt.Sprintf("%s-local", instance.Name): Equal(fmt.Sprintf("%s-local", instance.Name)), - }), - }), - }), - "PriorityClassName": Equal(*instance.Spec.PriorityClassName), - "Containers": MatchAllElements(testutils.ContainerIterator, Elements{ - common.Etcd: MatchFields(IgnoreExtras, Fields{ - "Ports": ConsistOf([]corev1.ContainerPort{ - { - Name: "server", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: *instance.Spec.Etcd.ServerPort, - }, - { - Name: "client", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: *instance.Spec.Etcd.ClientPort, - }, - }), - "Args": MatchAllElements(testutils.CmdIterator, Elements{ - "start-etcd": Equal("start-etcd"), - "--backup-restore-tls-enabled=true": Equal("--backup-restore-tls-enabled=true"), - "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt": Equal("--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt"), - "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key": Equal("--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key"), - "--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/ca.crt": Equal("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/ca.crt"), - fmt.Sprintf("--backup-restore-host-port=%s-local:%d", instance.Name, backupPort): Equal(fmt.Sprintf("--backup-restore-host-port=%s-local:%d", instance.Name, backupPort)), - fmt.Sprintf("--etcd-server-name=%s-local", instance.Name): Equal(fmt.Sprintf("--etcd-server-name=%s-local", instance.Name)), - }), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "Image": Equal(*instance.Spec.Etcd.Image), - "Resources": MatchFields(IgnoreExtras, Fields{ - "Requests": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceCPU: Equal(instance.Spec.Etcd.Resources.Requests[corev1.ResourceCPU]), - corev1.ResourceMemory: Equal(instance.Spec.Etcd.Resources.Requests[corev1.ResourceMemory]), - }), - "Limits": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceCPU: Equal(instance.Spec.Etcd.Resources.Limits[corev1.ResourceCPU]), - corev1.ResourceMemory: Equal(instance.Spec.Etcd.Resources.Limits[corev1.ResourceMemory]), - }), - }), - "ReadinessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ - "ProbeHandler": MatchFields(IgnoreExtras, Fields{ - "HTTPGet": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal("/healthz"), - "Port": Equal(intstr.FromInt(int(backupPort))), - "Scheme": Equal(corev1.URISchemeHTTPS), - })), - }), - "InitialDelaySeconds": Equal(int32(15)), - "PeriodSeconds": Equal(int32(5)), - "FailureThreshold": Equal(int32(5)), - })), - "VolumeMounts": MatchAllElements(testutils.VolumeMountIterator, Elements{ - *instance.Spec.VolumeClaimTemplate: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(*instance.Spec.VolumeClaimTemplate), - "MountPath": Equal("/var/etcd/data/"), - }), - "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-ca-etcd"), - "MountPath": Equal("/var/etcd/ssl/client/ca"), - }), - "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-server-tls"), - "MountPath": Equal("/var/etcd/ssl/client/server"), - }), - "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-client-tls"), - "MountPath": Equal("/var/etcd/ssl/client/client"), - }), - "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-ca-etcd"), - "MountPath": Equal("/var/etcd/ssl/peer/ca"), - }), - "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-etcd-server-tls"), - "MountPath": Equal("/var/etcd/ssl/peer/server"), - }), - }), - }), - - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchAllElements(testutils.CmdIterator, Elements{ - "server": Equal("server"), - "--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"), - "--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"), - "--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"), - "--server-cert=/var/etcd/ssl/client/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/client/server/tls.crt"), - "--server-key=/var/etcd/ssl/client/server/tls.key": Equal("--server-key=/var/etcd/ssl/client/server/tls.key"), - "--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"), - "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp"), - "--insecure-transport=false": Equal("--insecure-transport=false"), - "--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"), - "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), - "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), - "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), - "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), - "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), - fmt.Sprintf("--defragmentation-schedule=%s", *instance.Spec.Etcd.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *instance.Spec.Etcd.DefragmentationSchedule)), - fmt.Sprintf("--schedule=%s", *instance.Spec.Backup.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *instance.Spec.Backup.FullSnapshotSchedule)), - fmt.Sprintf("%s=%s", "--garbage-collection-policy", *instance.Spec.Backup.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *instance.Spec.Backup.GarbageCollectionPolicy)), - fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - fmt.Sprintf("--delta-snapshot-memory-limit=%d", instance.Spec.Backup.DeltaSnapshotMemoryLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", instance.Spec.Backup.DeltaSnapshotMemoryLimit.Value())), - fmt.Sprintf("--garbage-collection-policy=%s", *instance.Spec.Backup.GarbageCollectionPolicy): Equal(fmt.Sprintf("--garbage-collection-policy=%s", *instance.Spec.Backup.GarbageCollectionPolicy)), - fmt.Sprintf("--endpoints=https://%s-local:%d", instance.Name, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", instance.Name, clientPort)), - fmt.Sprintf("--service-endpoints=https://%s:%d", instance.GetClientServiceName(), clientPort): Equal(fmt.Sprintf("--service-endpoints=https://%s:%d", instance.GetClientServiceName(), clientPort)), - fmt.Sprintf("--embedded-etcd-quota-bytes=%d", instance.Spec.Etcd.Quota.Value()): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", instance.Spec.Etcd.Quota.Value())), - fmt.Sprintf("%s=%s", "--delta-snapshot-period", instance.Spec.Backup.DeltaSnapshotPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-period", instance.Spec.Backup.DeltaSnapshotPeriod.Duration.String())), - fmt.Sprintf("%s=%s", "--garbage-collection-period", instance.Spec.Backup.GarbageCollectionPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-period", instance.Spec.Backup.GarbageCollectionPeriod.Duration.String())), - fmt.Sprintf("%s=%s", "--auto-compaction-mode", *instance.Spec.Common.AutoCompactionMode): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-mode", autoCompactionMode)), - fmt.Sprintf("%s=%s", "--auto-compaction-retention", *instance.Spec.Common.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", autoCompactionRetention)), - fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", instance.Spec.Backup.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", instance.Spec.Backup.EtcdSnapshotTimeout.Duration.String())), - fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", instance.Spec.Etcd.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", instance.Spec.Etcd.EtcdDefragTimeout.Duration.String())), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", instance.GetDeltaSnapshotLeaseName()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", instance.GetDeltaSnapshotLeaseName())), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", instance.GetFullSnapshotLeaseName()): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", instance.GetFullSnapshotLeaseName())), - }), - "Ports": ConsistOf([]corev1.ContainerPort{ - { - Name: "server", - Protocol: corev1.ProtocolTCP, - HostPort: 0, - ContainerPort: backupPort, - }, - }), - "Image": Equal(*instance.Spec.Backup.Image), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "VolumeMounts": MatchElements(testutils.VolumeMountIterator, IgnoreExtras, Elements{ - *instance.Spec.VolumeClaimTemplate: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(*instance.Spec.VolumeClaimTemplate), - "MountPath": Equal("/var/etcd/data"), - }), - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "MountPath": Equal("/var/etcd/config/"), - }), - "host-storage": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("host-storage"), - "MountPath": Equal("/home/nonroot/" + *instance.Spec.Backup.Store.Container), - }), - }), - "Env": MatchElements(testutils.EnvIterator, IgnoreExtras, Elements{ - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - }), - "SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{ - "Capabilities": PointTo(MatchFields(IgnoreExtras, Fields{ - "Add": ConsistOf([]corev1.Capability{ - "SYS_PTRACE", - }), - })), - })), - }), - }), - "ShareProcessNamespace": Equal(pointer.Bool(true)), - "Volumes": MatchAllElements(testutils.VolumeIterator, Elements{ - "host-storage": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("host-storage"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "HostPath": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal(fmt.Sprintf("/etc/gardener/local-backupbuckets/%s", *instance.Spec.Backup.Store.Container)), - "Type": PointTo(Equal(corev1.HostPathType("Directory"))), - })), - }), - }), - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ - "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(fmt.Sprintf("etcd-bootstrap-%s", string(instance.UID[:6]))), - }), - "DefaultMode": PointTo(Equal(int32(0644))), - "Items": MatchAllElements(testutils.KeyIterator, Elements{ - "etcd.conf.yaml": MatchFields(IgnoreExtras, Fields{ - "Key": Equal("etcd.conf.yaml"), - "Path": Equal("etcd.conf.yaml"), - }), - }), - })), - }), - }), - "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-server-tls"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), - })), - }), - }), - "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-client-tls"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), - })), - }), - }), - "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-ca-etcd"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), - })), - }), - }), - "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-etcd-server-tls"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), - })), - }), - }), - "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-ca-etcd"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), - })), - }), - }), - }), - }), - }), - "VolumeClaimTemplates": MatchAllElements(testutils.PVCIterator, Elements{ - *instance.Spec.VolumeClaimTemplate: MatchFields(IgnoreExtras, Fields{ - "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(*instance.Spec.VolumeClaimTemplate), - }), - "Spec": MatchFields(IgnoreExtras, Fields{ - "StorageClassName": PointTo(Equal(*instance.Spec.StorageClass)), - "AccessModes": MatchAllElements(testutils.AccessModeIterator, Elements{ - "ReadWriteOnce": Equal(corev1.ReadWriteOnce), - }), - "Resources": MatchFields(IgnoreExtras, Fields{ - "Requests": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceStorage: Equal(*instance.Spec.StorageCapacity), - }), - }), - }), - }), - }), - }), - })) -} - -func validateStoreGCP(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, _ *corev1.ConfigMap, _ *corev1.Service, _ *corev1.Service) { - - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchElements(testutils.ContainerIterator, IgnoreExtras, Elements{ - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchElements(testutils.CmdIterator, IgnoreExtras, Elements{ - "--storage-provider=GCS": Equal("--storage-provider=GCS"), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - }), - "VolumeMounts": MatchElements(testutils.VolumeMountIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "MountPath": Equal("/var/.gcp/"), - }), - }), - "Env": MatchAllElements(testutils.EnvIterator, Elements{ - common.EnvStorageContainer: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*instance.Spec.Backup.Store.Container), - }), - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - common.EnvGoogleApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvGoogleApplicationCredentials), - "Value": Equal("/var/.gcp/serviceaccount.json"), - }), - common.EnvGoogleStorageAPIEndpoint: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvGoogleStorageAPIEndpoint), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretKeyRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.Spec.Backup.Store.SecretRef.Name), - }), - "Key": Equal("storageAPIEndpoint"), - "Optional": Equal(pointer.Bool(true)), - })), - })), - }), - }), - }), - }), - "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - })), - }), - }), - }), - }), - }), - }), - })) - -} - -func validateStoreAzure(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, _ *corev1.ConfigMap, _ *corev1.Service, _ *corev1.Service) { - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchElements(testutils.ContainerIterator, IgnoreExtras, Elements{ - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchElements(testutils.CmdIterator, IgnoreExtras, Elements{ - "--storage-provider=ABS": Equal("--storage-provider=ABS"), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - }), - "Env": MatchAllElements(testutils.EnvIterator, Elements{ - common.EnvStorageContainer: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*instance.Spec.Backup.Store.Container), - }), - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvAzureApplicationCredentials), - "Value": Equal("/var/etcd-backup"), - }), - }), - }), - }), - "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - })), - }), - }), - }), - }), - }), - }), - })) -} - -func validateStoreOpenstack(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, _ *corev1.ConfigMap, _ *corev1.Service, _ *corev1.Service) { - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchElements(testutils.ContainerIterator, IgnoreExtras, Elements{ - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchElements(testutils.CmdIterator, IgnoreExtras, Elements{ - "--storage-provider=Swift": Equal("--storage-provider=Swift"), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - }), - "Env": MatchAllElements(testutils.EnvIterator, Elements{ - common.EnvStorageContainer: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*instance.Spec.Backup.Store.Container), - }), - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - common.EnvOpenstackApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvOpenstackApplicationCredentials), - "Value": Equal("/var/etcd-backup"), - }), - }), - }), - }), - "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - })), - }), - }), - }), - }), - }), - }), - })) -} - -func validateStoreAlicloud(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, _ *corev1.ConfigMap, _ *corev1.Service, _ *corev1.Service) { - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchElements(testutils.ContainerIterator, IgnoreExtras, Elements{ - - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchElements(testutils.CmdIterator, IgnoreExtras, Elements{ - "--storage-provider=OSS": Equal("--storage-provider=OSS"), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - }), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "Env": MatchAllElements(testutils.EnvIterator, Elements{ - common.EnvStorageContainer: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*instance.Spec.Backup.Store.Container), - }), - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - common.EnvAlicloudApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvAlicloudApplicationCredentials), - "Value": Equal("/var/etcd-backup"), - }), - }), - }), - }), - "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - })), - }), - }), - }), - }), - }), - }), - })) -} - -func validateStoreAWS(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, _ *corev1.ConfigMap, _ *corev1.Service, _ *corev1.Service) { - Expect(*s).To(MatchFields(IgnoreExtras, Fields{ - "Spec": MatchFields(IgnoreExtras, Fields{ - "Template": MatchFields(IgnoreExtras, Fields{ - //s.Spec.Template.Spec.HostAliases - "Spec": MatchFields(IgnoreExtras, Fields{ - "Containers": MatchElements(testutils.ContainerIterator, IgnoreExtras, Elements{ - - backupRestore: MatchFields(IgnoreExtras, Fields{ - "Args": MatchElements(testutils.CmdIterator, IgnoreExtras, Elements{ - "--storage-provider=S3": Equal("--storage-provider=S3"), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - }), - "ImagePullPolicy": Equal(corev1.PullIfNotPresent), - "Env": MatchAllElements(testutils.EnvIterator, Elements{ - common.EnvStorageContainer: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvStorageContainer), - "Value": Equal(*instance.Spec.Backup.Store.Container), - }), - common.EnvPodName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodName), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.name"), - })), - })), - }), - common.EnvPodNamespace: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvPodNamespace), - "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ - "FieldPath": Equal("metadata.namespace"), - })), - })), - }), - common.EnvAWSApplicationCredentials: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EnvAWSApplicationCredentials), - "Value": Equal("/var/etcd-backup"), - }), - }), - }), - }), - "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "VolumeSource": MatchFields(IgnoreExtras, Fields{ - "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - })), - }), - }), - }), - }), - }), - }), - })) -} - -var _ = Describe("buildPredicate", func() { - var ( - instance *druidv1alpha1.Etcd - evalCreate = func(p predicate.Predicate, obj client.Object) bool { return p.Create(event.CreateEvent{Object: obj}) } - evalDelete = func(p predicate.Predicate, obj client.Object) bool { return p.Delete(event.DeleteEvent{Object: obj}) } - evalGeneric = func(p predicate.Predicate, obj client.Object) bool { return p.Generic(event.GenericEvent{Object: obj}) } - evalUpdateWithoutGenerationChange = func(p predicate.Predicate, obj client.Object) bool { - return p.Update(event.UpdateEvent{ObjectOld: obj, ObjectNew: obj.DeepCopyObject().(client.Object)}) - } - evalUpdateWithGenerationChange = func(p predicate.Predicate, obj client.Object) bool { - objCopy := obj.DeepCopyObject().(client.Object) - objCopy.SetGeneration(obj.GetGeneration() + 1) - return p.Update(event.UpdateEvent{ObjectOld: obj, ObjectNew: objCopy}) - } - ) - - BeforeEach(func() { - instance = &druidv1alpha1.Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{}, - }, - } - }) - - DescribeTable( - "with ignoreOperationAnnotation true", - func(evalFn func(p predicate.Predicate, obj client.Object) bool, expect bool) { - Expect(evalFn(etcd.BuildPredicate(true), instance)).To(Equal(expect)) - }, - Entry("Create should match", evalCreate, true), - Entry("Delete should match", evalDelete, true), - Entry("Generic should match", evalGeneric, true), - Entry("Update without generation change should not match", evalUpdateWithoutGenerationChange, false), - Entry("Update with generation change should match", evalUpdateWithGenerationChange, true), - ) - - Describe("with ignoreOperationAnnotation false", func() { - DescribeTable( - "without operation annotation or last error or deletion timestamp", - func(evalFn func(p predicate.Predicate, obj client.Object) bool, expect bool) { - Expect(evalFn(etcd.BuildPredicate(false), instance)).To(Equal(expect)) - }, - Entry("Create should not match", evalCreate, false), - Entry("Delete should match", evalDelete, true), - Entry("Generic should not match", evalGeneric, false), - Entry("Update without generation change should not match", evalUpdateWithoutGenerationChange, false), - Entry("Update with generation change should not match", evalUpdateWithGenerationChange, false), - ) - DescribeTable( - "with operation annotation", - func(evalFn func(p predicate.Predicate, obj client.Object) bool, expect bool) { - instance.Annotations[v1beta1constants.GardenerOperation] = v1beta1constants.GardenerOperationReconcile - Expect(evalFn(etcd.BuildPredicate(false), instance)).To(Equal(expect)) - }, - Entry("Create should match", evalCreate, true), - Entry("Delete should match", evalDelete, true), - Entry("Generic should match", evalGeneric, true), - Entry("Update without generation change should match", evalUpdateWithoutGenerationChange, true), - Entry("Update with generation change should match", evalUpdateWithGenerationChange, true), - ) - DescribeTable( - "with last error", - func(evalFn func(p predicate.Predicate, obj client.Object) bool, expect bool) { - instance.Status.LastError = pointer.String("error") - Expect(evalFn(etcd.BuildPredicate(false), instance)).To(Equal(expect)) - }, - Entry("Create should match", evalCreate, true), - Entry("Delete should match", evalDelete, true), - Entry("Generic should match", evalGeneric, true), - Entry("Update without generation change should match", evalUpdateWithoutGenerationChange, true), - Entry("Update with generation change should match", evalUpdateWithGenerationChange, true), - ) - DescribeTable( - "with deletion timestamp", - func(evalFn func(p predicate.Predicate, obj client.Object) bool, expect bool) { - now := metav1.Time{Time: time.Now()} - instance.DeletionTimestamp = &now - Expect(evalFn(etcd.BuildPredicate(false), instance)).To(Equal(expect)) - }, - Entry("Create should match", evalCreate, true), - Entry("Delete should match", evalDelete, true), - Entry("Generic should match", evalGeneric, true), - Entry("Update without generation change should match", evalUpdateWithoutGenerationChange, true), - Entry("Update with generation change should match", evalUpdateWithGenerationChange, true), - ) - }) -}) diff --git a/test/integration/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go b/test/integration/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go index 067d95505..93167c936 100644 --- a/test/integration/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go +++ b/test/integration/controllers/etcdcopybackupstask/etcdcopybackupstask_suite_test.go @@ -7,7 +7,7 @@ package etcdcopybackupstask import ( "testing" - "github.com/gardener/etcd-druid/controllers/etcdcopybackupstask" + "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/test/integration/controllers/assets" "github.com/gardener/etcd-druid/test/integration/setup" "github.com/gardener/gardener/pkg/utils/imagevector" diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index cc98ec820..404dda2c3 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -10,8 +10,8 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" - "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" diff --git a/test/integration/controllers/secret/reconciler_test.go b/test/integration/controllers/secret/reconciler_test.go index f320a63d8..0d4b946da 100644 --- a/test/integration/controllers/secret/reconciler_test.go +++ b/test/integration/controllers/secret/reconciler_test.go @@ -9,7 +9,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/utils/test/matchers" diff --git a/test/integration/controllers/secret/secret_suite_test.go b/test/integration/controllers/secret/secret_suite_test.go index fc2bb5451..b56f8e064 100644 --- a/test/integration/controllers/secret/secret_suite_test.go +++ b/test/integration/controllers/secret/secret_suite_test.go @@ -8,7 +8,7 @@ import ( "context" "testing" - "github.com/gardener/etcd-druid/controllers/secret" + "github.com/gardener/etcd-druid/internal/controller/secret" "github.com/gardener/etcd-druid/test/integration/controllers/assets" "github.com/gardener/etcd-druid/test/integration/setup" . "github.com/onsi/ginkgo/v2" From ea474be25cf92fa0c93048ab760699e80d8094c7 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 15 Mar 2024 09:38:36 +0530 Subject: [PATCH 102/235] Run `make revendor` --- go.mod | 6 +++--- test/it/controller/assets/assets.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 59cd12e78..6790cba16 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.22 require ( github.com/gardener/etcd-backup-restore v0.26.0 github.com/gardener/gardener v1.86.0 - github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 @@ -13,9 +12,7 @@ require ( github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 github.com/prometheus/client_golang v1.16.0 - github.com/robfig/cron/v3 v3.0.1 github.com/spf13/pflag v1.0.5 - go.uber.org/mock v0.2.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 gopkg.in/yaml.v2 v2.4.0 @@ -61,6 +58,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gardener/hvpa-controller/api v0.5.0 // indirect github.com/gardener/machine-controller-manager v0.50.0 // indirect + github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -103,12 +101,14 @@ require ( github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cobra v1.7.0 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 // indirect go.opencensus.io v0.24.0 // indirect + go.uber.org/mock v0.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.12.0 // indirect diff --git a/test/it/controller/assets/assets.go b/test/it/controller/assets/assets.go index 3c805dc1f..bd62035ce 100644 --- a/test/it/controller/assets/assets.go +++ b/test/it/controller/assets/assets.go @@ -7,7 +7,7 @@ package assets import ( "path/filepath" - "github.com/gardener/etcd-druid/pkg/common" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/gomega" From 93f6422dd68dd885cc9861070f7ee706bfb4a038 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 15 Mar 2024 11:46:26 +0530 Subject: [PATCH 103/235] fixed etcdcopybackupstasktest and minor refactoring --- internal/controller/etcd/config.go | 6 ++-- .../etcdcopybackupstask/reconciler_test.go | 4 --- internal/health/status/check_test.go | 10 +++--- internal/utils/miscellaneous.go | 32 ++++--------------- test/it/controller/etcd/reconciler_test.go | 2 +- 5 files changed, 14 insertions(+), 40 deletions(-) diff --git a/internal/controller/etcd/config.go b/internal/controller/etcd/config.go index 307da8e97..62f805333 100644 --- a/internal/controller/etcd/config.go +++ b/internal/controller/etcd/config.go @@ -63,11 +63,11 @@ type Config struct { FeatureGates map[featuregate.Feature]bool // EtcdMember holds configuration related to etcd members. - EtcdMember EtcdMemberConfig + EtcdMember MemberConfig } -// EtcdMemberConfig holds configuration related to etcd members. -type EtcdMemberConfig struct { +// MemberConfig holds configuration related to etcd members. +type MemberConfig struct { // NotReadyThreshold is the duration after which an etcd member's state is considered `NotReady`. NotReadyThreshold time.Duration // UnknownThreshold is the duration after which an etcd member's state is considered `Unknown`. diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 59c300321..f0d3b8bf2 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -714,10 +714,6 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", task.Namespace, task.Name)), - "gardener.cloud/owner-type": Equal("etcdcopybackupstask"), - }), "OwnerReferences": MatchAllElements(testutils.OwnerRefIterator, Elements{ task.Name: MatchAllFields(Fields{ "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), diff --git a/internal/health/status/check_test.go b/internal/health/status/check_test.go index 94abe91b8..c9f765e27 100644 --- a/internal/health/status/check_test.go +++ b/internal/health/status/check_test.go @@ -12,6 +12,7 @@ import ( "github.com/gardener/etcd-druid/internal/health/condition" "github.com/gardener/etcd-druid/internal/health/etcdmember" . "github.com/gardener/etcd-druid/internal/health/status" + "github.com/gardener/etcd-druid/internal/utils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,9 +29,6 @@ import ( var _ = Describe("Check", func() { Describe("#Check", func() { It("should correctly execute checks and fill status", func() { - memberRoleLeader := druidv1alpha1.EtcdRoleLeader - memberRoleMember := druidv1alpha1.EtcdRoleMember - timeBefore, _ := time.Parse(time.RFC3339, "2021-06-01T00:00:00Z") timeNow := timeBefore.Add(1 * time.Hour) @@ -111,9 +109,9 @@ var _ = Describe("Check", func() { defer test.WithVar(&EtcdMemberChecks, []EtcdMemberCheckFn{ func(_ client.Client, _ logr.Logger, _, _ time.Duration) etcdmember.Checker { return createEtcdMemberCheck( - etcdMemberResult{pointer.String("1"), "member1", &memberRoleLeader, druidv1alpha1.EtcdMemberStatusUnknown, "Unknown"}, - etcdMemberResult{pointer.String("2"), "member2", &memberRoleMember, druidv1alpha1.EtcdMemberStatusNotReady, "bar reason"}, - etcdMemberResult{pointer.String("3"), "member3", &memberRoleMember, druidv1alpha1.EtcdMemberStatusReady, "foobar reason"}, + etcdMemberResult{pointer.String("1"), "member1", utils.PointerOf[druidv1alpha1.EtcdRole](druidv1alpha1.EtcdRoleLeader), druidv1alpha1.EtcdMemberStatusUnknown, "Unknown"}, + etcdMemberResult{pointer.String("2"), "member2", utils.PointerOf[druidv1alpha1.EtcdRole](druidv1alpha1.EtcdRoleMember), druidv1alpha1.EtcdMemberStatusNotReady, "bar reason"}, + etcdMemberResult{pointer.String("3"), "member3", utils.PointerOf[druidv1alpha1.EtcdRole](druidv1alpha1.EtcdRoleMember), druidv1alpha1.EtcdMemberStatusReady, "foobar reason"}, ) }, })() diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index 02df1097e..8e4b11f1b 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -15,26 +15,9 @@ import ( // MergeStringMaps merges the content of the newMaps with the oldMap. If a key already exists then // it gets overwritten by the last value with the same key. func MergeStringMaps(oldMap map[string]string, newMaps ...map[string]string) map[string]string { - var out map[string]string - - if oldMap != nil { - out = make(map[string]string) - } - for k, v := range oldMap { - out[k] = v - } - - for _, newMap := range newMaps { - if newMap != nil && out == nil { - out = make(map[string]string) - } - - for k, v := range newMap { - out[k] = v - } - } - - return out + allMaps := []map[string]string{oldMap} + allMaps = append(allMaps, newMaps...) + return MergeMaps[string, string](allMaps...) } // MergeMaps merges the contents of maps. All maps will be processed in the order @@ -99,10 +82,7 @@ func IfConditionOr[T any](condition bool, trueVal, falseVal T) T { return falseVal } -// IsNilOrEmptyStringPtr returns true if the string pointer is nil or the return value of IsEmptyString(s). -func IsNilOrEmptyStringPtr(s *string) bool { - if s == nil { - return true - } - return IsEmptyString(*s) +// PointerOf returns a pointer to the given value. +func PointerOf[T any](val T) *T { + return &val } diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 6c12a3a34..67a193475 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -474,7 +474,7 @@ func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTe FeatureGates: map[featuregate.Feature]bool{ features.UseEtcdWrapper: true, }, - EtcdMember: etcd.EtcdMemberConfig{ + EtcdMember: etcd.MemberConfig{ NotReadyThreshold: 5 * time.Minute, UnknownThreshold: 1 * time.Minute, }, From 3c02f3a495d583c28790647cecdf8170523f22b1 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 15 Mar 2024 09:44:54 +0530 Subject: [PATCH 104/235] Run `make fmt` --- internal/mock/controller-runtime/client/doc.go | 1 - internal/operator/registry.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/mock/controller-runtime/client/doc.go b/internal/mock/controller-runtime/client/doc.go index cf1678271..44a22b400 100644 --- a/internal/mock/controller-runtime/client/doc.go +++ b/internal/mock/controller-runtime/client/doc.go @@ -2,5 +2,4 @@ // // SPDX-License-Identifier: Apache-2.0 - package client diff --git a/internal/operator/registry.go b/internal/operator/registry.go index 08c00b9e5..a43f7c5d4 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -66,4 +66,4 @@ func (r registry) GetOperator(kind Kind) component.Operator { func (r registry) AllOperators() map[Kind]component.Operator { return r.operators -} \ No newline at end of file +} From b8fd1865a637cf7b3ccf9c520787f3817c6918cc Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 15 Mar 2024 12:12:23 +0530 Subject: [PATCH 105/235] Fix `make check`; fix and run `make generate` --- .github/dependabot.yml | 4 -- Makefile | 6 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 4 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 4 +- config/rbac/role.yaml | 33 ++--------- hack/addlicenseheaders.sh | 14 ++--- hack/boilerplate.go.txt | 6 +- hack/update-codegen.sh | 3 +- internal/controller/etcd/reconcile_status.go | 2 +- internal/controller/etcd/reconciler.go | 14 +++-- internal/controller/utils/etcdstatus.go | 1 + internal/controller/utils/reconciler.go | 13 +++++ .../operator/clientservice/clientservice.go | 23 +++++--- internal/operator/component/types.go | 1 + internal/operator/configmap/configmap.go | 23 +++++--- internal/operator/memberlease/memberlease.go | 23 +++++--- internal/operator/peerservice/peerservice.go | 23 +++++--- .../poddisruptionbudget.go | 23 +++++--- internal/operator/registry.go | 1 + internal/operator/role/role.go | 23 +++++--- internal/operator/rolebinding/rolebinding.go | 57 +++++++++++-------- .../operator/serviceaccount/serviceaccount.go | 25 +++++--- .../operator/snapshotlease/snapshotlease.go | 23 +++++--- internal/operator/statefulset/builder.go | 6 +- internal/operator/statefulset/statefulset.go | 16 ++++-- internal/operator/statefulset/stsmatcher.go | 4 +- internal/utils/miscellaneous.go | 6 +- internal/webhook/sentinel/config.go | 1 + internal/webhook/sentinel/handler.go | 3 +- 29 files changed, 222 insertions(+), 163 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e5d42e3d5..da0acb85e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,3 @@ -# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -# -# SPDX-License-Identifier: Apache-2.0 - # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: diff --git a/Makefile b/Makefile index 6523b9fcb..49cf112d9 100644 --- a/Makefile +++ b/Makefile @@ -88,9 +88,9 @@ check-generate: # Generate code .PHONY: generate -generate: manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) - @go generate "$(REPO_ROOT)/pkg/..." - @bash $(HACK_DIR)/update-codegen.sh +generate: set-permissions manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) + @go generate "$(REPO_ROOT)/internal/..." + @"$(REPO_ROOT)/hack/update-codegen.sh" # Build the docker image .PHONY: docker-build diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index cfccfd992..05a339d99 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -213,8 +213,8 @@ spec: type: string type: object maxBackupsLimitBasedGC: - description: MaxBackupsLimitBasedGC defines the maximum number - of Full snapshots to retain in Limit Based GarbageCollectionPolicy + description: |- + MaxBackupsLimitBasedGC defines the maximum number of Full snapshots to retain in Limit Based GarbageCollectionPolicy All full snapshots beyond this limit will be garbage collected. format: int32 type: integer diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index cfccfd992..05a339d99 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -213,8 +213,8 @@ spec: type: string type: object maxBackupsLimitBasedGC: - description: MaxBackupsLimitBasedGC defines the maximum number - of Full snapshots to retain in Limit Based GarbageCollectionPolicy + description: |- + MaxBackupsLimitBasedGC defines the maximum number of Full snapshots to retain in Limit Based GarbageCollectionPolicy All full snapshots beyond this limit will be garbage collected. format: int32 type: integer diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 888072d47..30d5dc344 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -17,17 +17,6 @@ rules: - list - patch - update - - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: @@ -36,9 +25,6 @@ rules: - create - get - list - - patch - - update - - watch - apiGroups: - "" resources: @@ -77,6 +63,12 @@ rules: - list - patch - update +- apiGroups: + - apps + resources: + - statefulsets/status + verbs: + - get - watch - apiGroups: - batch @@ -131,21 +123,11 @@ rules: - etcds verbs: - create - - delete - get - list - patch - update - watch -- apiGroups: - - druid.gardener.cloud - resources: - - etcds/finalizers - verbs: - - create - - get - - patch - - update - apiGroups: - druid.gardener.cloud resources: @@ -166,7 +148,6 @@ rules: - list - patch - update - - watch - apiGroups: - rbac.authorization.k8s.io resources: @@ -178,7 +159,6 @@ rules: - list - patch - update - - watch - apiGroups: - rbac.authorization.k8s.io resources: @@ -190,4 +170,3 @@ rules: - list - patch - update - - watch diff --git a/hack/addlicenseheaders.sh b/hack/addlicenseheaders.sh index f753069c2..92de4d909 100755 --- a/hack/addlicenseheaders.sh +++ b/hack/addlicenseheaders.sh @@ -8,26 +8,20 @@ set -e echo "> Adding Apache License header to all go files where it is not present" -YEAR=$1 -if [[ -z "$1" ]]; then - cat << EOF -Unspecified 'YEAR' argument. -Usage: addlicenceheaders.sh -EOF - exit 1 -fi +YEAR="$(date +%Y)" -# addlicence with a license file (parameter -f) expects no comments in the file. +# addlicense with a license file (parameter -f) expects no comments in the file. # boilerplate.go.txt is however also used also when generating go code. # Therefore we remove '//' from boilerplate.go.txt here before passing it to addlicense. temp_file=$(mktemp) trap "rm -f $temp_file" EXIT -sed "s/{YEAR}/${YEAR}/g" hack/boilerplate.go.txt > $temp_file +sed -e "s/YEAR/${YEAR}/g" -e 's|^// *||' hack/boilerplate.go.txt > $temp_file addlicense \ -f $temp_file \ -ignore "**/*.md" \ -ignore "**/*.yaml" \ + -ignore "**/*.yml" \ -ignore "**/Dockerfile" \ . \ No newline at end of file diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 9266b594b..4fcf3e9b7 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,3 +1,3 @@ -SPDX-FileCopyrightText: {YEAR} SAP SE or an SAP affiliate company and Gardener contributors - -SPDX-License-Identifier: Apache-2.0 \ No newline at end of file +// SPDX-FileCopyrightText: YEAR SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index f68d75955..e88ac4500 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -12,7 +12,8 @@ PROJECT_DIR="$(dirname "$SCRIPT_DIR")" echo "> Update Code Generation" -cd "$PROJECT_DIR/api" && controller-gen "object:headerFile=$SCRIPT_DIR/boilerplate.go.txt" paths=./... +YEAR="$(date +%Y)" +cd "$PROJECT_DIR/api" && controller-gen "object:headerFile=$SCRIPT_DIR/boilerplate.go.txt,year=${YEAR}" paths=./... # needed as long as https://github.com/kubernetes-sigs/controller-tools/issues/559 is not fixed find "$PROJECT_DIR/api" -type f -name "zz_*.go" -exec goimports -w '{}' \; diff --git a/internal/controller/etcd/reconcile_status.go b/internal/controller/etcd/reconcile_status.go index afaefe32b..08032d804 100644 --- a/internal/controller/etcd/reconcile_status.go +++ b/internal/controller/etcd/reconcile_status.go @@ -50,7 +50,7 @@ func (r *Reconciler) mutateETCDStatusWithMemberStatusAndConditions(ctx component return ctrlutils.ContinueReconcile() } -func (r *Reconciler) inspectStatefulSetAndMutateETCDStatus(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, logger logr.Logger) ctrlutils.ReconcileStepResult { +func (r *Reconciler) inspectStatefulSetAndMutateETCDStatus(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, _ logr.Logger) ctrlutils.ReconcileStepResult { sts, err := utils.GetStatefulSet(ctx, r.client, etcd) if err != nil { return ctrlutils.ReconcileWithError(err) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index be2fc1e9b..807b8ac79 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) +// Reconciler reconciles the Etcd resource spec and status. type Reconciler struct { client client.Client config *Config @@ -51,6 +52,7 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { return NewReconcilerWithImageVector(mgr, config, imageVector) } +// NewReconcilerWithImageVector creates a new reconciler for Etcd with the given image vector. func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, iv imagevector.ImageVector) (*Reconciler, error) { logger := log.Log.WithName(controllerName) operatorReg := createAndInitializeOperatorRegistry(mgr.GetClient(), config, iv) @@ -71,12 +73,12 @@ type reconcileFn func(ctx component.OperatorContext, objectKey client.ObjectKey) // TODO: where/how is this being used? // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds/status,verbs=get;create;update;patch -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update;patch;triggerDeletionFlow -// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;create;update;patch;triggerDeletionFlow -// +kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;create;update;patch;triggerDeletionFlow -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;create;update;patch;triggerDeletionFlow -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;create;update;patch;triggerDeletionFlow -// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;triggerDeletionFlow +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;watch // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=events,verbs=create;get;list diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index e67935328..274777062 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -22,6 +22,7 @@ type LastOperationErrorRecorder interface { RecordError(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error } +// NewLastOperationErrorRecorder returns a new LastOperationErrorRecorder func NewLastOperationErrorRecorder(client client.Client, logger logr.Logger) LastOperationErrorRecorder { return &lastOpErrRecorder{ client: client, diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 9185c04af..e568223b9 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -53,6 +53,7 @@ func ContainsFinalizer(o client.Object, finalizer string) bool { return false } +// GetLatestEtcd returns the latest version of the Etcd object. func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ReconcileStepResult { if err := client.Get(ctx, objectKey, etcd); err != nil { if apierrors.IsNotFound(err) { @@ -63,6 +64,7 @@ func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.O return ContinueReconcile() } +// ReconcileStepResult holds the result of a reconcile step. type ReconcileStepResult struct { result ctrl.Result errs []error @@ -70,22 +72,27 @@ type ReconcileStepResult struct { continueReconcile bool } +// ReconcileResult returns the result and error from the reconcile step. func (r ReconcileStepResult) ReconcileResult() (ctrl.Result, error) { return r.result, errors.Join(r.errs...) } +// GetErrors returns the errors from the reconcile step. func (r ReconcileStepResult) GetErrors() []error { return r.errs } +// GetResult returns the result from the reconcile step. func (r ReconcileStepResult) GetResult() ctrl.Result { return r.result } +// HasErrors returns true if there are errors from the reconcile step. func (r ReconcileStepResult) HasErrors() bool { return len(r.errs) > 0 } +// GetDescription returns the description of the reconcile step. func (r ReconcileStepResult) GetDescription() string { if len(r.errs) > 0 { return fmt.Sprintf("%s %s", r.description, errors.Join(r.errs...).Error()) @@ -93,6 +100,7 @@ func (r ReconcileStepResult) GetDescription() string { return r.description } +// DoNotRequeue returns a ReconcileStepResult that does not requeue the reconciliation. func DoNotRequeue() ReconcileStepResult { return ReconcileStepResult{ continueReconcile: false, @@ -100,12 +108,14 @@ func DoNotRequeue() ReconcileStepResult { } } +// ContinueReconcile returns a ReconcileStepResult that continues the reconciliation. func ContinueReconcile() ReconcileStepResult { return ReconcileStepResult{ continueReconcile: true, } } +// ReconcileWithError returns a ReconcileStepResult with the given errors. func ReconcileWithError(errs ...error) ReconcileStepResult { return ReconcileStepResult{ continueReconcile: false, @@ -114,6 +124,7 @@ func ReconcileWithError(errs ...error) ReconcileStepResult { } } +// ReconcileAfter returns a ReconcileStepResult that requeues the reconciliation after the given period. func ReconcileAfter(period time.Duration, description string) ReconcileStepResult { return ReconcileStepResult{ continueReconcile: false, @@ -122,6 +133,7 @@ func ReconcileAfter(period time.Duration, description string) ReconcileStepResul } } +// ReconcileWithErrorAfter returns a ReconcileStepResult that requeues the reconciliation after the given period with the given errors. func ReconcileWithErrorAfter(period time.Duration, errs ...error) ReconcileStepResult { return ReconcileStepResult{ result: ctrl.Result{RequeueAfter: period}, @@ -130,6 +142,7 @@ func ReconcileWithErrorAfter(period time.Duration, errs ...error) ReconcileStepR } } +// ShortCircuitReconcileFlow indicates whether to short-circuit the reconciliation. func ShortCircuitReconcileFlow(result ReconcileStepResult) bool { return !result.continueReconcile } diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 8cc6c8ae4..f70b6541b 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -28,15 +28,26 @@ const ( ) const ( - ErrGetClientService druidv1alpha1.ErrorCode = "ERR_GET_CLIENT_SERVICE" + // ErrGetClientService indicates an error in getting the client service resource. + ErrGetClientService druidv1alpha1.ErrorCode = "ERR_GET_CLIENT_SERVICE" + // ErrSyncClientService indicates an error in syncing the client service resource. + ErrSyncClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" + // ErrDeleteClientService indicates an error in deleting the client service resource. ErrDeleteClientService druidv1alpha1.ErrorCode = "ERR_DELETE_CLIENT_SERVICE" - ErrSyncClientService druidv1alpha1.ErrorCode = "ERR_SYNC_CLIENT_SERVICE" ) type _resource struct { client client.Client } +// New returns a new client service operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the name of the existing client service for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) svc := &corev1.Service{} @@ -56,6 +67,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the client service for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) svc := emptyClientService(objectKey) @@ -74,6 +86,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// TriggerDelete triggers the deletion of the client service for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of client service", "objectKey", objectKey) @@ -93,12 +106,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { svc.Labels = getLabels(etcd) svc.Annotations = getAnnotations(etcd) diff --git a/internal/operator/component/types.go b/internal/operator/component/types.go index 3d4a560ec..aad60fb20 100644 --- a/internal/operator/component/types.go +++ b/internal/operator/component/types.go @@ -32,6 +32,7 @@ func NewOperatorContext(ctx context.Context, logger logr.Logger, runID string) O } } +// SetLogger sets the logger for the OperatorContext. func (o *OperatorContext) SetLogger(logger logr.Logger) { o.Logger = logger } diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 5fa199c0d..52fb3628e 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -23,9 +23,12 @@ import ( ) const ( - ErrGetConfigMap druidv1alpha1.ErrorCode = "ERR_GET_CONFIGMAP" + // ErrGetConfigMap indicates an error in getting the configmap resource. + ErrGetConfigMap druidv1alpha1.ErrorCode = "ERR_GET_CONFIGMAP" + // ErrSyncConfigMap indicates an error in syncing the configmap resource. + ErrSyncConfigMap druidv1alpha1.ErrorCode = "ERR_SYNC_CONFIGMAP" + // ErrDeleteConfigMap indicates an error in deleting the configmap resource. ErrDeleteConfigMap druidv1alpha1.ErrorCode = "ERR_DELETE_CONFIGMAP" - ErrSyncConfigMap druidv1alpha1.ErrorCode = "ERR_SYNC_CONFIGMAP" ) const etcdConfigKey = "etcd.conf.yaml" @@ -34,6 +37,14 @@ type _resource struct { client client.Client } +// New returns a new configmap operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the name of the existing configmap for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objKey := getObjectKey(etcd) @@ -53,6 +64,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the configmap for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { cm := emptyConfigMap(getObjectKey(etcd)) result, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { @@ -76,6 +88,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// TriggerDelete triggers the deletion of the configmap for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) @@ -95,12 +108,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { cfg := createEtcdConfig(etcd) cfgYaml, err := yaml.Marshal(cfg) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index c12f3408d..4ca26f12a 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -21,15 +21,26 @@ import ( ) const ( - ErrListMemberLease druidv1alpha1.ErrorCode = "ERR_LIST_MEMBER_LEASE" + // ErrListMemberLease indicates an error in listing the member lease resources. + ErrListMemberLease druidv1alpha1.ErrorCode = "ERR_LIST_MEMBER_LEASE" + // ErrSyncMemberLease indicates an error in syncing the member lease resources. + ErrSyncMemberLease druidv1alpha1.ErrorCode = "ERR_SYNC_MEMBER_LEASE" + // ErrDeleteMemberLease indicates an error in deleting the member lease resources. ErrDeleteMemberLease druidv1alpha1.ErrorCode = "ERR_DELETE_MEMBER_LEASE" - ErrSyncMemberLease druidv1alpha1.ErrorCode = "ERR_SYNC_MEMBER_LEASE" ) type _resource struct { client client.Client } +// New returns a new member lease operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the names of the existing member leases for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) leaseList := &coordinationv1.LeaseList{} @@ -51,6 +62,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the member leases for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKeys := getObjectKeys(etcd) createTasks := make([]utils.OperatorTask, len(objectKeys)) @@ -89,6 +101,7 @@ func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1 return nil } +// TriggerDelete deletes the member leases for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of member leases") if err := r.client.DeleteAllOf(ctx, @@ -104,12 +117,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { lease.Labels = getLabels(etcd, lease.Name) lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 9fd82d215..3a7f253e2 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -23,15 +23,26 @@ import ( const defaultServerPort = 2380 const ( - ErrGetPeerService druidv1alpha1.ErrorCode = "ERR_GET_PEER_SERVICE" + // ErrGetPeerService indicates an error in getting the peer service resource. + ErrGetPeerService druidv1alpha1.ErrorCode = "ERR_GET_PEER_SERVICE" + // ErrSyncPeerService indicates an error in syncing the peer service resource. + ErrSyncPeerService druidv1alpha1.ErrorCode = "ERR_SYNC_PEER_SERVICE" + // ErrDeletePeerService indicates an error in deleting the peer service resource. ErrDeletePeerService druidv1alpha1.ErrorCode = "ERR_DELETE_PEER_SERVICE" - ErrSyncPeerService druidv1alpha1.ErrorCode = "ERR_SYNC_PEER_SERVICE" ) type _resource struct { client client.Client } +// New returns a new peer service operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the name of the existing peer service for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) svcObjectKey := getObjectKey(etcd) @@ -51,6 +62,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the peer service for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) svc := emptyPeerService(objectKey) @@ -69,6 +81,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// TriggerDelete triggers the deletion of the peer service for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of peer service") @@ -89,12 +102,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { svc.Labels = getLabels(etcd) svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index c7557623f..993909161 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -21,15 +21,26 @@ import ( ) const ( - ErrGetPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_GET_POD_DISRUPTION_BUDGET" + // ErrGetPodDisruptionBudget indicates an error in getting the pod disruption budget resource. + ErrGetPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_GET_POD_DISRUPTION_BUDGET" + // ErrSyncPodDisruptionBudget indicates an error in syncing the pod disruption budget resource. + ErrSyncPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_SYNC_POD_DISRUPTION_BUDGET" + // ErrDeletePodDisruptionBudget indicates an error in deleting the pod disruption budget resource. ErrDeletePodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_DELETE_POD_DISRUPTION_BUDGET" - ErrSyncPodDisruptionBudget druidv1alpha1.ErrorCode = "ERR_SYNC_POD_DISRUPTION_BUDGET" ) type _resource struct { client client.Client } +// New returns a new pod disruption budget operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the name of the existing pod disruption budget for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) @@ -49,6 +60,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the pod disruption budget for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) pdb := emptyPodDisruptionBudget(objectKey) @@ -67,6 +79,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// TriggerDelete triggers the deletion of the pod disruption budget for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of PDB") pdbObjectKey := getObjectKey(etcd) @@ -80,12 +93,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) { pdb.Labels = getLabels(etcd) pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} diff --git a/internal/operator/registry.go b/internal/operator/registry.go index a43f7c5d4..434bf2b5e 100644 --- a/internal/operator/registry.go +++ b/internal/operator/registry.go @@ -19,6 +19,7 @@ type Registry interface { GetOperator(kind Kind) component.Operator } +// Kind represents the kind of component that an operator manages. type Kind string const ( diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 431c57a3b..a9dfb2284 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -21,8 +21,11 @@ import ( ) const ( - ErrGetRole druidv1alpha1.ErrorCode = "ERR_GET_ROLE" - ErrSyncRole druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE" + // ErrGetRole indicates an error in getting the role resource. + ErrGetRole druidv1alpha1.ErrorCode = "ERR_GET_ROLE" + // ErrSyncRole indicates an error in syncing the role resource. + ErrSyncRole druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE" + // ErrDeleteRole indicates an error in deleting the role resource. ErrDeleteRole druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE" ) @@ -30,6 +33,14 @@ type _resource struct { client client.Client } +// New returns a new role operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the name of the existing role for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) @@ -49,6 +60,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the role for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) role := emptyRole(objectKey) @@ -67,6 +79,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// TriggerDelete triggers the deletion of the role for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) @@ -85,12 +98,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { return client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace} } diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index e843dcec9..d034d4ec7 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -21,8 +21,11 @@ import ( ) const ( - ErrGetRoleBinding druidv1alpha1.ErrorCode = "ERR_GET_ROLE_BINDING" - ErrSyncRoleBinding druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE_BINDING" + // ErrGetRoleBinding indicates an error in getting the role binding resource. + ErrGetRoleBinding druidv1alpha1.ErrorCode = "ERR_GET_ROLE_BINDING" + // ErrSyncRoleBinding indicates an error in syncing the role binding resource. + ErrSyncRoleBinding druidv1alpha1.ErrorCode = "ERR_SYNC_ROLE_BINDING" + // ErrDeleteRoleBinding indicates an error in deleting the role binding resource. ErrDeleteRoleBinding druidv1alpha1.ErrorCode = "ERR_DELETE_ROLE_BINDING" ) @@ -30,6 +33,14 @@ type _resource struct { client client.Client } +// New returns a new role binding operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the name of the existing role binding for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) @@ -49,25 +60,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) - err := r.client.Delete(ctx, emptyRoleBinding(objectKey)) - if err != nil { - if errors.IsNotFound(err) { - ctx.Logger.Info("No RoleBinding found, Deletion is a No-Op", "objectKey", objectKey) - return nil - } - return druiderr.WrapError(err, - ErrDeleteRoleBinding, - "TriggerDelete", - fmt.Sprintf("Failed to delete role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), - ) - } - ctx.Logger.Info("deleted", "component", "role-binding", "objectKey", objectKey) - return nil -} - +// Sync creates or updates the role binding for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) rb := emptyRoleBinding(objectKey) @@ -86,6 +79,8 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// Exists checks if the role binding exists for the given Etcd. +// TODO: This method is not used. Remove it if not needed. func (r _resource) Exists(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { rb := &rbacv1.RoleBinding{} if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { @@ -97,10 +92,24 @@ func (r _resource) Exists(ctx component.OperatorContext, etcd *druidv1alpha1.Etc return true, nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, +// TriggerDelete triggers the deletion of the role binding for the given Etcd. +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { + objectKey := getObjectKey(etcd) + ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) + err := r.client.Delete(ctx, emptyRoleBinding(objectKey)) + if err != nil { + if errors.IsNotFound(err) { + ctx.Logger.Info("No RoleBinding found, Deletion is a No-Op", "objectKey", objectKey) + return nil + } + return druiderr.WrapError(err, + ErrDeleteRoleBinding, + "TriggerDelete", + fmt.Sprintf("Failed to delete role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + ) } + ctx.Logger.Info("deleted", "component", "role-binding", "objectKey", objectKey) + return nil } func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 8d1532c41..43887c36c 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -21,9 +21,12 @@ import ( ) const ( - ErrGetServiceAccount druidv1alpha1.ErrorCode = "ERR_GET_SERVICE_ACCOUNT" + // ErrGetServiceAccount indicates an error in getting the service account resource. + ErrGetServiceAccount druidv1alpha1.ErrorCode = "ERR_GET_SERVICE_ACCOUNT" + // ErrSyncServiceAccount indicates an error in syncing the service account resource. + ErrSyncServiceAccount druidv1alpha1.ErrorCode = "ERR_SYNC_SERVICE_ACCOUNT" + // ErrDeleteServiceAccount indicates an error in deleting the service account resource. ErrDeleteServiceAccount druidv1alpha1.ErrorCode = "ERR_DELETE_SERVICE_ACCOUNT" - ErrSyncServiceAccount druidv1alpha1.ErrorCode = "ERR_SYNC_SERVICE_ACCOUNT" ) type _resource struct { @@ -31,6 +34,15 @@ type _resource struct { disableAutoMount bool } +// New returns a new service account operator. +func New(client client.Client, disableAutomount bool) component.Operator { + return &_resource{ + client: client, + disableAutoMount: disableAutomount, + } +} + +// GetExistingResourceNames returns the name of the existing service account for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) sa := &corev1.ServiceAccount{} @@ -50,6 +62,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the service account for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) sa := emptyServiceAccount(objectKey) @@ -68,6 +81,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } +// TriggerDelete triggers the deletion of the service account for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of service account") objectKey := getObjectKey(etcd) @@ -85,13 +99,6 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func New(client client.Client, disableAutomount bool) component.Operator { - return &_resource{ - client: client, - disableAutoMount: disableAutomount, - } -} - func buildResource(etcd *druidv1alpha1.Etcd, sa *corev1.ServiceAccount, autoMountServiceAccountToken bool) { sa.Labels = getLabels(etcd) sa.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 149c0015e..a4fc6f8f9 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -22,15 +22,26 @@ import ( ) const ( - ErrGetSnapshotLease druidv1alpha1.ErrorCode = "ERR_GET_SNAPSHOT_LEASE" + // ErrGetSnapshotLease indicates an error in getting the snapshot lease resources. + ErrGetSnapshotLease druidv1alpha1.ErrorCode = "ERR_GET_SNAPSHOT_LEASE" + // ErrSyncSnapshotLease indicates an error in syncing the snapshot lease resources. + ErrSyncSnapshotLease druidv1alpha1.ErrorCode = "ERR_SYNC_SNAPSHOT_LEASE" + // ErrDeleteSnapshotLease indicates an error in deleting the snapshot lease resources. ErrDeleteSnapshotLease druidv1alpha1.ErrorCode = "ERR_DELETE_SNAPSHOT_LEASE" - ErrSyncSnapshotLease druidv1alpha1.ErrorCode = "ERR_SYNC_SNAPSHOT_LEASE" ) type _resource struct { client client.Client } +// New returns a new snapshot lease operator. +func New(client client.Client) component.Operator { + return &_resource{ + client: client, + } +} + +// GetExistingResourceNames returns the names of the existing snapshot leases for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 2) // We have to get snapshot leases one lease at a time and cannot use label-selector based listing @@ -66,6 +77,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the snapshot leases for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { ctx.Logger.Info("Backup has been disabled. Triggering delete of snapshot leases") @@ -92,6 +104,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return errors.Join(utils.RunConcurrently(ctx, syncTasks)...) } +// TriggerDelete triggers the deletion of the snapshot leases for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { ctx.Logger.Info("Triggering delete of snapshot leases") if err := r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { @@ -116,12 +129,6 @@ func (r _resource) deleteAllSnapshotLeases(ctx component.OperatorContext, etcd * return nil } -func New(client client.Client) component.Operator { - return &_resource{ - client: client, - } -} - func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*coordinationv1.Lease, error) { lease := &coordinationv1.Lease{} if err := r.client.Get(ctx, objectKey, lease); err != nil { diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index d8f3d1427..c43ae0f30 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -105,12 +105,10 @@ func newStsBuilder(client client.Client, }, nil } +// Build builds the StatefulSet for the given Etcd. func (b *stsBuilder) Build(ctx component.OperatorContext) error { b.createStatefulSetObjectMeta() - if err := b.createStatefulSetSpec(ctx); err != nil { - return err - } - return nil + return b.createStatefulSetSpec(ctx) } func (b *stsBuilder) createStatefulSetObjectMeta() { diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 208573b38..988302110 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -24,9 +24,12 @@ import ( ) const ( - ErrGetStatefulSet druidv1alpha1.ErrorCode = "ERR_GET_STATEFULSET" + // ErrGetStatefulSet indicates an error in getting the statefulset resource. + ErrGetStatefulSet druidv1alpha1.ErrorCode = "ERR_GET_STATEFULSET" + // ErrSyncStatefulSet indicates an error in syncing the statefulset resource. + ErrSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_SYNC_STATEFULSET" + // ErrDeleteStatefulSet indicates an error in deleting the statefulset resource. ErrDeleteStatefulSet druidv1alpha1.ErrorCode = "ERR_DELETE_STATEFULSET" - ErrSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_SYNC_STATEFULSET" ) type _resource struct { @@ -35,6 +38,7 @@ type _resource struct { useEtcdWrapper bool } +// New returns a new statefulset resource. func New(client client.Client, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) component.Operator { return &_resource{ client: client, @@ -43,6 +47,7 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates } } +// GetExistingResourceNames returns the name of the existing statefulset for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) @@ -62,6 +67,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, nil } +// Sync creates or updates the statefulset for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { var ( existingSTS *appsv1.StatefulSet @@ -90,6 +96,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return r.createOrPatch(ctx, etcd) } +// TriggerDelete triggers the deletion of the statefulset for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of StatefulSet", "objectKey", objectKey) @@ -211,6 +218,7 @@ func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druid return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } +// TODO: this function is not used. Remove it if not needed. func deleteAllStsPods(ctx component.OperatorContext, cl client.Client, opName string, sts *appsv1.StatefulSet) error { // Get all Pods belonging to the StatefulSet podList := &corev1.PodList{} @@ -220,13 +228,13 @@ func deleteAllStsPods(ctx component.OperatorContext, cl client.Client, opName st } if err := cl.List(ctx, podList, listOpts...); err != nil { - ctx.Logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts)) + ctx.Logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts), "operation", opName) return err } for _, pod := range podList.Items { if err := cl.Delete(ctx, &pod); err != nil { - ctx.Logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace) + ctx.Logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace, "operation", opName) return err } } diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index efa8dded6..b1c1e3c3e 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -236,7 +236,7 @@ func (s StatefulSetMatcher) matchEtcdContainerVolMounts() gomegatypes.GomegaMatc volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) secretVolMountMatchers := s.getSecretVolMountsMatchers() - if secretVolMountMatchers != nil && len(secretVolMountMatchers) > 0 { + if len(secretVolMountMatchers) > 0 { volMountMatchers = append(volMountMatchers, secretVolMountMatchers...) } return ConsistOf(volMountMatchers) @@ -425,7 +425,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { }) volMatchers = append(volMatchers, etcdConfigFileVolMountMatcher) secretVolMatchers := s.getPodSecurityVolumeMatchers() - if secretVolMatchers != nil && len(secretVolMatchers) > 0 { + if len(secretVolMatchers) > 0 { volMatchers = append(volMatchers, secretVolMatchers...) } if s.etcd.IsBackupStoreEnabled() { diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index 8e4b11f1b..f25db2e16 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -67,11 +67,9 @@ func TypeDeref[T any](val *T, defaultVal T) T { return defaultVal } +// IsEmptyString returns true if the string is empty or contains only whitespace characters. func IsEmptyString(s string) bool { - if len(strings.TrimSpace(s)) == 0 { - return true - } - return false + return len(strings.TrimSpace(s)) == 0 } // IfConditionOr implements a simple ternary operator, if the passed condition is true then trueVal is returned else falseVal is returned. diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/sentinel/config.go index f25d3b44f..51c165fcf 100644 --- a/internal/webhook/sentinel/config.go +++ b/internal/webhook/sentinel/config.go @@ -6,6 +6,7 @@ package sentinel import ( "fmt" + flag "github.com/spf13/pflag" ) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 5a9d7995b..564375f98 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -// Handler handles admission requests and prevents changes to resources created by etcd-druid. +// Handler is the Sentinel Webhook admission handler. type Handler struct { client.Client config *Config @@ -51,6 +51,7 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { }, nil } +// Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { var ( requestGK = schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} From 2ec2afadf43c66ec776d168c36d2567c10916c44 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 15 Mar 2024 17:53:16 +0530 Subject: [PATCH 106/235] Add unit tests for DataVolumesReady condition checker --- .../condition/check_data_volumes_ready.go | 15 +- .../check_data_volumes_ready_test.go | 248 ++++++++++++++++++ 2 files changed, 255 insertions(+), 8 deletions(-) create mode 100644 internal/health/condition/check_data_volumes_ready_test.go diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go index b2ba8c76d..06091102e 100644 --- a/internal/health/condition/check_data_volumes_ready.go +++ b/internal/health/condition/check_data_volumes_ready.go @@ -8,11 +8,11 @@ import ( "context" "fmt" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/utils" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -28,17 +28,16 @@ func (d *dataVolumesReady) Check(ctx context.Context, etcd druidv1alpha1.Etcd) R sts, err := utils.GetStatefulSet(ctx, d.cl, &etcd) if err != nil { + if errors.IsNotFound(err) { + res.reason = "StatefulSetNotFound" + res.message = fmt.Sprintf("StatefulSet %s not found for etcd", etcd.Name) + return res + } res.reason = "UnableToFetchStatefulSet" res.message = fmt.Sprintf("Unable to fetch StatefulSet for etcd: %s", err.Error()) return res } - if sts == nil { - res.reason = "StatefulSetNotFound" - res.message = fmt.Sprintf("StatefulSet: %q not found for etcd", etcd.Name) - return res - } - pvcEvents, err := utils.FetchPVCWarningMessageForStatefulSet(ctx, d.cl, sts) if err != nil { res.reason = "UnableToFetchWarningEventsForDataVolumes" diff --git a/internal/health/condition/check_data_volumes_ready_test.go b/internal/health/condition/check_data_volumes_ready_test.go new file mode 100644 index 000000000..b83739a00 --- /dev/null +++ b/internal/health/condition/check_data_volumes_ready_test.go @@ -0,0 +1,248 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package condition_test + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/health/condition" + mockclient "github.com/gardener/etcd-druid/internal/mock/controller-runtime/client" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("DataVolumesReadyCheck", func() { + Describe("#Check", func() { + var ( + mockCtrl *gomock.Controller + cl *mockclient.MockClient + + notFoundErr = apierrors.StatusError{ + ErrStatus: v1.Status{ + Reason: v1.StatusReasonNotFound, + }, + } + internalErr = apierrors.StatusError{ + ErrStatus: v1.Status{ + Reason: v1.StatusReasonInternalError, + }, + } + + etcd = druidv1alpha1.Etcd{ + ObjectMeta: v1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: druidv1alpha1.EtcdSpec{ + Replicas: 1, + }, + } + sts = &appsv1.StatefulSet{ + ObjectMeta: v1.ObjectMeta{ + Name: "test", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: pointer.Int32(1), + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "test", + }, + }, + }, + }, + Status: appsv1.StatefulSetStatus{ + CurrentReplicas: 1, + ReadyReplicas: 1, + }, + } + pvc = &corev1.PersistentVolumeClaim{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-test-0", + Namespace: "default", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + } + event = &corev1.Event{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-event", + Namespace: "default", + }, + InvolvedObject: corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: "test-pvc", + Namespace: "default", + }, + Reason: "FailedMount", + Message: "MountVolume.SetUp failed for volume \"test-pvc\" : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name csi.gardener.cloud not found in the list of registered CSI drivers", + Type: corev1.EventTypeWarning, + } + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + cl = mockclient.NewMockClient(mockCtrl) + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + Context("when error in fetching statefulset", func() { + It("should return that the condition is unknown", func() { + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.StatefulSetList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, stsList *appsv1.StatefulSetList, _ ...client.ListOption) error { + *stsList = appsv1.StatefulSetList{ + Items: []appsv1.StatefulSet{}, + } + return &internalErr + }).AnyTimes() + + check := condition.DataVolumesReadyCheck(cl) + result := check.Check(context.Background(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeDataVolumesReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal("UnableToFetchStatefulSet")) + }) + }) + + Context("when statefulset not found", func() { + It("should return that the condition is unknown", func() { + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.StatefulSetList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, stsList *appsv1.StatefulSetList, _ ...client.ListOption) error { + *stsList = appsv1.StatefulSetList{ + Items: []appsv1.StatefulSet{}, + } + return ¬FoundErr + }).AnyTimes() + + check := condition.DataVolumesReadyCheck(cl) + result := check.Check(context.Background(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeDataVolumesReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal("StatefulSetNotFound")) + }) + }) + + Context("when error in fetching warning events for PVCs", func() { + It("should return that the condition is unknown", func() { + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.StatefulSetList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, stsList *appsv1.StatefulSetList, _ ...client.ListOption) error { + *stsList = appsv1.StatefulSetList{ + Items: []appsv1.StatefulSet{*sts}, + } + return nil + }).AnyTimes() + + cl.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ client.ObjectList, _ ...client.ListOption) error { + return &internalErr + }).AnyTimes() + + check := condition.DataVolumesReadyCheck(cl) + result := check.Check(context.Background(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeDataVolumesReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionUnknown)) + Expect(result.Reason()).To(Equal("UnableToFetchWarningEventsForDataVolumes")) + }) + }) + + Context("when warning events found for PVCs", func() { + It("should return that the condition is false", func() { + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.StatefulSetList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, stsList *appsv1.StatefulSetList, _ ...client.ListOption) error { + *stsList = appsv1.StatefulSetList{ + Items: []appsv1.StatefulSet{*sts}, + } + return nil + }).AnyTimes() + + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&corev1.PersistentVolumeClaimList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, pvcs *corev1.PersistentVolumeClaimList, _ ...client.ListOption) error { + *pvcs = corev1.PersistentVolumeClaimList{ + Items: []corev1.PersistentVolumeClaim{*pvc}, + } + return nil + }).AnyTimes() + + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + cl.EXPECT().Scheme().Return(scheme).AnyTimes() + + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&corev1.EventList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, eventsList *corev1.EventList, _ ...client.ListOption) error { + *eventsList = corev1.EventList{ + Items: []corev1.Event{*event}, + } + return nil + }).AnyTimes() + + check := condition.DataVolumesReadyCheck(cl) + result := check.Check(context.Background(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeDataVolumesReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionFalse)) + Expect(result.Reason()).To(Equal("FoundWarningsForDataVolumes")) + }) + }) + + Context("when no warning events found for PVCs", func() { + It("should return that the condition is true", func() { + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.StatefulSetList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, stsList *appsv1.StatefulSetList, _ ...client.ListOption) error { + *stsList = appsv1.StatefulSetList{ + Items: []appsv1.StatefulSet{*sts}, + } + return nil + }).AnyTimes() + + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&corev1.PersistentVolumeClaimList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, pvcs *corev1.PersistentVolumeClaimList, _ ...client.ListOption) error { + *pvcs = corev1.PersistentVolumeClaimList{ + Items: []corev1.PersistentVolumeClaim{*pvc}, + } + return nil + }).AnyTimes() + + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + cl.EXPECT().Scheme().Return(scheme).AnyTimes() + + cl.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&corev1.EventList{}), gomock.Any()).DoAndReturn( + func(_ context.Context, eventsList *corev1.EventList, _ ...client.ListOption) error { + *eventsList = corev1.EventList{ + Items: []corev1.Event{}, + } + return nil + }).AnyTimes() + + check := condition.DataVolumesReadyCheck(cl) + result := check.Check(context.Background(), etcd) + + Expect(result.ConditionType()).To(Equal(druidv1alpha1.ConditionTypeDataVolumesReady)) + Expect(result.Status()).To(Equal(druidv1alpha1.ConditionTrue)) + Expect(result.Reason()).To(Equal("NoWarningsFoundForDataVolumes")) + }) + }) + }) +}) From 062c50b0ce1ac4894fb931074d9b4cca2118e06d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 19 Mar 2024 11:16:10 +0530 Subject: [PATCH 107/235] fixed unit tests, introduced constants and adapted Makefile --- Makefile | 5 +- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 342 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 3616 ++++++++--------- internal/operator/statefulset/builder.go | 80 +- internal/operator/statefulset/constants.go | 15 + internal/operator/statefulset/stsmatcher.go | 110 +- 6 files changed, 2093 insertions(+), 2075 deletions(-) diff --git a/Makefile b/Makefile index 49cf112d9..9988950fb 100644 --- a/Makefile +++ b/Makefile @@ -107,14 +107,17 @@ docker-push: # Run tests .PHONY: test test: $(GINKGO) + # run ginkgo unit tests. These will be ported to golang native tests over a period of time. @"$(REPO_ROOT)/hack/test.sh" ./api/... \ ./internal/controller/etcdcopybackupstask/... \ ./internal/controller/predicate/... \ ./internal/controller/secret/... \ ./internal/controller/utils/... \ - ./internal/health/... \ ./internal/mapper/... \ ./internal/metrics/... + # run the golang native unit tests. + @go test -v -coverprofile cover.out ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... + @go tool cover -func=cover.out .PHONY: test-cov test-cov: $(GINKGO) $(SETUP_ENVTEST) diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 9aa0b085e..87ccdf510 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -14,183 +14,183 @@ spec: singular: etcdcopybackupstask scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: EtcdCopyBackupsTask is a task for copying etcd backups from a - source to a target store. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: EtcdCopyBackupsTaskSpec defines the parameters for the copy - backups task. - properties: - maxBackupAge: - description: |- - MaxBackupAge is the maximum age in days that a backup must have in order to be copied. - By default all backups will be copied. - format: int32 - type: integer - maxBackups: - description: MaxBackups is the maximum number of backups that will - be copied starting with the most recent ones. - format: int32 - type: integer - sourceStore: - description: SourceStore defines the specification of the source object - store provider for storing backups. - properties: - container: - description: Container is the name of the container the backup - is stored at. - type: string - prefix: - description: Prefix is the prefix used for the store. - type: string - provider: - description: Provider is the name of the backup provider. - type: string - secretRef: - description: SecretRef is the reference to the secret which used - to connect to the backup store. - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which the - secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - prefix - type: object - targetStore: - description: TargetStore defines the specification of the target object - store provider for storing backups. - properties: - container: - description: Container is the name of the container the backup - is stored at. - type: string - prefix: - description: Prefix is the prefix used for the store. - type: string - provider: - description: Provider is the name of the backup provider. - type: string - secretRef: - description: SecretRef is the reference to the secret which used - to connect to the backup store. - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which the - secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - prefix - type: object - waitForFinalSnapshot: - description: WaitForFinalSnapshot defines the parameters for waiting - for a final full snapshot before copying backups. - properties: - enabled: - description: Enabled specifies whether to wait for a final full - snapshot before copying backups. - type: boolean - timeout: - description: |- - Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups - will be performed anyway. No timeout or 0 means wait forever. - type: string - required: - - enabled - type: object - required: - - sourceStore - - targetStore - type: object - status: - description: EtcdCopyBackupsTaskStatus defines the observed state of the - copy backups task. - properties: - conditions: - description: Conditions represents the latest available observations - of an object's current state. - items: - description: Condition holds the information about the state of - a resource. + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: EtcdCopyBackupsTask is a task for copying etcd backups from a + source to a target store. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EtcdCopyBackupsTaskSpec defines the parameters for the copy + backups task. + properties: + maxBackupAge: + description: |- + MaxBackupAge is the maximum age in days that a backup must have in order to be copied. + By default all backups will be copied. + format: int32 + type: integer + maxBackups: + description: MaxBackups is the maximum number of backups that will + be copied starting with the most recent ones. + format: int32 + type: integer + sourceStore: + description: SourceStore defines the specification of the source object + store provider for storing backups. properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. - format: date-time + container: + description: Container is the name of the container the backup + is stored at. type: string - lastUpdateTime: - description: Last time the condition was updated. - format: date-time + prefix: + description: Prefix is the prefix used for the store. type: string - message: - description: A human readable message indicating details about - the transition. + provider: + description: Provider is the name of the backup provider. type: string - reason: - description: The reason for the condition's last transition. + secretRef: + description: SecretRef is the reference to the secret which used + to connect to the backup store. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - prefix + type: object + targetStore: + description: TargetStore defines the specification of the target object + store provider for storing backups. + properties: + container: + description: Container is the name of the container the backup + is stored at. type: string - status: - description: Status of the condition, one of True, False, Unknown. + prefix: + description: Prefix is the prefix used for the store. type: string - type: - description: Type of the Etcd condition. + provider: + description: Provider is the name of the backup provider. type: string + secretRef: + description: SecretRef is the reference to the secret which used + to connect to the backup store. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic required: - - lastTransitionTime - - lastUpdateTime - - message - - reason - - status - - type + - prefix type: object - type: array - lastError: - description: LastError represents the last occurred error. - type: string - observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this resource. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} + waitForFinalSnapshot: + description: WaitForFinalSnapshot defines the parameters for waiting + for a final full snapshot before copying backups. + properties: + enabled: + description: Enabled specifies whether to wait for a final full + snapshot before copying backups. + type: boolean + timeout: + description: |- + Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups + will be performed anyway. No timeout or 0 means wait forever. + type: string + required: + - enabled + type: object + required: + - sourceStore + - targetStore + type: object + status: + description: EtcdCopyBackupsTaskStatus defines the observed state of the + copy backups task. + properties: + conditions: + description: Conditions represents the latest available observations + of an object's current state. + items: + description: Condition holds the information about the state of + a resource. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: Last time the condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of the Etcd condition. + type: string + required: + - lastTransitionTime + - lastUpdateTime + - message + - reason + - status + - type + type: object + type: array + lastError: + description: LastError represents the last occurred error. + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this resource. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 05a339d99..fdbea37a9 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -14,1930 +14,1894 @@ spec: singular: etcd scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .status.ready - name: Ready - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Quorate - type: string - - jsonPath: .status.conditions[?(@.type=="AllMembersReady")].status - name: All Members Ready - type: string - - jsonPath: .status.conditions[?(@.type=="BackupReady")].status - name: Backup Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .spec.replicas - name: Cluster Size - priority: 1 - type: integer - - jsonPath: .status.currentReplicas - name: Current Replicas - priority: 1 - type: integer - - jsonPath: .status.readyReplicas - name: Ready Replicas - priority: 1 - type: integer - name: v1alpha1 - schema: - openAPIV3Schema: - description: Etcd is the Schema for the etcds API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: EtcdSpec defines the desired state of Etcd - properties: - annotations: - additionalProperties: - type: string - type: object - backup: - description: BackupSpec defines parameters associated with the full - and delta snapshots of etcd. - properties: - compactionResources: - description: |- - CompactionResources defines compute Resources required by compaction job. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. + - additionalPrinterColumns: + - jsonPath: .status.ready + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Quorate + type: string + - jsonPath: .status.conditions[?(@.type=="AllMembersReady")].status + name: All Members Ready + type: string + - jsonPath: .status.conditions[?(@.type=="BackupReady")].status + name: Backup Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.replicas + name: Cluster Size + priority: 1 + type: integer + - jsonPath: .status.currentReplicas + name: Current Replicas + priority: 1 + type: integer + - jsonPath: .status.readyReplicas + name: Ready Replicas + priority: 1 + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: Etcd is the Schema for the etcds API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EtcdSpec defines the desired state of Etcd + properties: + annotations: + additionalProperties: + type: string + type: object + backup: + description: BackupSpec defines parameters associated with the full + and delta snapshots of etcd. + properties: + compactionResources: + description: |- + CompactionResources defines compute Resources required by compaction job. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + compression: + description: SnapshotCompression defines the specification for + compression of Snapshots. + properties: + enabled: + type: boolean + policy: + description: CompressionPolicy defines the type of policy + for compression of snapshots. + enum: + - gzip + - lzw + - zlib + type: string + type: object + deltaSnapshotMemoryLimit: + anyOf: + - type: integer + - type: string + description: DeltaSnapshotMemoryLimit defines the memory limit + after which delta snapshots will be taken + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + deltaSnapshotPeriod: + description: DeltaSnapshotPeriod defines the period after which + delta snapshots will be taken + type: string + deltaSnapshotRetentionPeriod: + description: |- + DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set. + The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d') + pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ + type: string + enableProfiling: + description: EnableProfiling defines if profiling should be enabled + for the etcd-backup-restore-sidecar + type: boolean + etcdSnapshotTimeout: + description: EtcdSnapshotTimeout defines the timeout duration + for etcd FullSnapshot operation + type: string + fullSnapshotSchedule: + description: FullSnapshotSchedule defines the cron standard schedule + for full snapshots. + type: string + garbageCollectionPeriod: + description: GarbageCollectionPeriod defines the period for garbage + collecting old backups + type: string + garbageCollectionPolicy: + description: GarbageCollectionPolicy defines the policy for garbage + collecting old backups + enum: + - Exponential + - LimitBased + type: string + image: + description: Image defines the etcd container image and tag + type: string + leaderElection: + description: LeaderElection defines parameters related to the + LeaderElection configuration. + properties: + etcdConnectionTimeout: + description: EtcdConnectionTimeout defines the timeout duration + for etcd client connection during leader election. + type: string + reelectionPeriod: + description: ReelectionPeriod defines the Period after which + leadership status of corresponding etcd is checked. + type: string + type: object + maxBackupsLimitBasedGC: + description: |- + MaxBackupsLimitBasedGC defines the maximum number of Full snapshots to retain in Limit Based GarbageCollectionPolicy + All full snapshots beyond this limit will be garbage collected. + format: int32 + type: integer + port: + description: Port define the port on which etcd-backup-restore + server will be exposed. + format: int32 + type: integer + resources: + description: |- + Resources defines compute Resources required by backup-restore container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + store: + description: Store defines the specification of object store provider + for storing backups. + properties: + container: + description: Container is the name of the container the backup + is stored at. + type: string + prefix: + description: Prefix is the prefix used for the store. + type: string + provider: + description: Provider is the name of the backup provider. + type: string + secretRef: + description: SecretRef is the reference to the secret which + used to connect to the backup store. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. type: string - required: - - name type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true -<<<<<<< HEAD - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. Requests cannot exceed - Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' -======= - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ->>>>>>> 8571402f (Fix and run `make generate`) - type: object - type: object - compression: - description: SnapshotCompression defines the specification for - compression of Snapshots. - properties: - enabled: - type: boolean - policy: - description: CompressionPolicy defines the type of policy - for compression of snapshots. - enum: - - gzip - - lzw - - zlib - type: string - type: object - deltaSnapshotMemoryLimit: - anyOf: - - type: integer - - type: string - description: DeltaSnapshotMemoryLimit defines the memory limit - after which delta snapshots will be taken - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - deltaSnapshotPeriod: - description: DeltaSnapshotPeriod defines the period after which - delta snapshots will be taken - type: string - deltaSnapshotRetentionPeriod: - description: |- - DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set. - The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d') - pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ - type: string - enableProfiling: - description: EnableProfiling defines if profiling should be enabled - for the etcd-backup-restore-sidecar - type: boolean - etcdSnapshotTimeout: - description: EtcdSnapshotTimeout defines the timeout duration - for etcd FullSnapshot operation - type: string - fullSnapshotSchedule: - description: FullSnapshotSchedule defines the cron standard schedule - for full snapshots. - type: string - garbageCollectionPeriod: - description: GarbageCollectionPeriod defines the period for garbage - collecting old backups - type: string - garbageCollectionPolicy: - description: GarbageCollectionPolicy defines the policy for garbage - collecting old backups - enum: - - Exponential - - LimitBased - type: string - image: - description: Image defines the etcd container image and tag - type: string - leaderElection: - description: LeaderElection defines parameters related to the - LeaderElection configuration. - properties: - etcdConnectionTimeout: - description: EtcdConnectionTimeout defines the timeout duration - for etcd client connection during leader election. - type: string - reelectionPeriod: - description: ReelectionPeriod defines the Period after which - leadership status of corresponding etcd is checked. - type: string - type: object - maxBackupsLimitBasedGC: - description: |- - MaxBackupsLimitBasedGC defines the maximum number of Full snapshots to retain in Limit Based GarbageCollectionPolicy - All full snapshots beyond this limit will be garbage collected. - format: int32 - type: integer - port: - description: Port define the port on which etcd-backup-restore - server will be exposed. - format: int32 - type: integer - resources: - description: |- - Resources defines compute Resources required by backup-restore container. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. + x-kubernetes-map-type: atomic + required: + - prefix + type: object + tls: + description: TLSConfig hold the TLS configuration details. + properties: + clientTLSSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. type: string - required: - - name type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true -<<<<<<< HEAD - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. Requests cannot exceed - Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' -======= - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ->>>>>>> 8571402f (Fix and run `make generate`) - type: object - type: object - store: - description: Store defines the specification of object store provider - for storing backups. - properties: - container: - description: Container is the name of the container the backup - is stored at. - type: string - prefix: - description: Prefix is the prefix used for the store. - type: string - provider: - description: Provider is the name of the backup provider. - type: string - secretRef: - description: SecretRef is the reference to the secret which - used to connect to the backup store. - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - prefix - type: object - tls: - description: TLSConfig hold the TLS configuration details. - properties: - clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - tlsCASecretRef: - description: SecretReference defines a reference to a secret. - properties: - dataKey: - description: DataKey is the name of the key in the data - map containing the credentials. - type: string - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - serverTLSSecretRef - - tlsCASecretRef - type: object - type: object - etcd: - description: EtcdConfig defines parameters associated etcd deployed - properties: - authSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which the - secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - clientPort: - format: int32 - type: integer - clientService: - description: ClientService defines the parameters of the client - service that a user can specify - properties: - annotations: - additionalProperties: + x-kubernetes-map-type: atomic + serverTLSSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + tlsCASecretRef: + description: SecretReference defines a reference to a secret. + properties: + dataKey: + description: DataKey is the name of the key in the data + map containing the credentials. + type: string + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - serverTLSSecretRef + - tlsCASecretRef + type: object + type: object + etcd: + description: EtcdConfig defines parameters associated etcd deployed + properties: + authSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. type: string - description: Annotations specify the annotations that should - be added to the client service - type: object - labels: - additionalProperties: + namespace: + description: namespace defines the space within which the + secret name must be unique. type: string - description: Labels specify the labels that should be added - to the client service - type: object - type: object - clientUrlTls: - description: ClientUrlTLS contains the ca, server TLS and client - TLS secrets for client communication to ETCD cluster - properties: - clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - tlsCASecretRef: - description: SecretReference defines a reference to a secret. - properties: - dataKey: - description: DataKey is the name of the key in the data - map containing the credentials. - type: string - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - serverTLSSecretRef - - tlsCASecretRef - type: object - defragmentationSchedule: - description: DefragmentationSchedule defines the cron standard - schedule for defragmentation of etcd. - type: string - etcdDefragTimeout: - description: EtcdDefragTimeout defines the timeout duration for - etcd defrag call - type: string - heartbeatDuration: - description: HeartbeatDuration defines the duration for members - to send heartbeats. The default value is 10s. - type: string - image: - description: Image defines the etcd container image and tag - type: string - metrics: - description: Metrics defines the level of detail for exported - metrics of etcd, specify 'extensive' to include histogram metrics. - enum: - - basic - - extensive - type: string - peerUrlTls: - description: |- - PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster - Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster. - properties: - clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - tlsCASecretRef: - description: SecretReference defines a reference to a secret. - properties: - dataKey: - description: DataKey is the name of the key in the data - map containing the credentials. - type: string - name: - description: name is unique within a namespace to reference - a secret resource. + type: object + x-kubernetes-map-type: atomic + clientPort: + format: int32 + type: integer + clientService: + description: ClientService defines the parameters of the client + service that a user can specify + properties: + annotations: + additionalProperties: type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. + description: Annotations specify the annotations that should + be added to the client service + type: object + labels: + additionalProperties: type: string - type: object - x-kubernetes-map-type: atomic - required: - - serverTLSSecretRef - - tlsCASecretRef - type: object - quota: - anyOf: - - type: integer - - type: string - description: Quota defines the etcd DB quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resources: - description: |- - Resources defines the compute Resources required by etcd container. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. + description: Labels specify the labels that should be added + to the client service + type: object + type: object + clientUrlTls: + description: ClientUrlTLS contains the ca, server TLS and client + TLS secrets for client communication to ETCD cluster + properties: + clientTLSSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. type: string - required: - - name type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true -<<<<<<< HEAD - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. Requests cannot exceed - Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' -======= - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ->>>>>>> 8571402f (Fix and run `make generate`) - type: object - type: object - serverPort: - format: int32 - type: integer - type: object - labels: - additionalProperties: + x-kubernetes-map-type: atomic + serverTLSSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + tlsCASecretRef: + description: SecretReference defines a reference to a secret. + properties: + dataKey: + description: DataKey is the name of the key in the data + map containing the credentials. + type: string + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - serverTLSSecretRef + - tlsCASecretRef + type: object + defragmentationSchedule: + description: DefragmentationSchedule defines the cron standard + schedule for defragmentation of etcd. + type: string + etcdDefragTimeout: + description: EtcdDefragTimeout defines the timeout duration for + etcd defrag call + type: string + heartbeatDuration: + description: HeartbeatDuration defines the duration for members + to send heartbeats. The default value is 10s. + type: string + image: + description: Image defines the etcd container image and tag + type: string + metrics: + description: Metrics defines the level of detail for exported + metrics of etcd, specify 'extensive' to include histogram metrics. + enum: + - basic + - extensive + type: string + peerUrlTls: + description: |- + PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster + Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster. + properties: + clientTLSSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + serverTLSSecretRef: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + tlsCASecretRef: + description: SecretReference defines a reference to a secret. + properties: + dataKey: + description: DataKey is the name of the key in the data + map containing the credentials. + type: string + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - serverTLSSecretRef + - tlsCASecretRef + type: object + quota: + anyOf: + - type: integer + - type: string + description: Quota defines the etcd DB quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resources: + description: |- + Resources defines the compute Resources required by etcd container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + serverPort: + format: int32 + type: integer + type: object + labels: + additionalProperties: + type: string + type: object + priorityClassName: + description: PriorityClassName is the name of a priority class that + shall be used for the etcd pods. type: string - type: object - priorityClassName: - description: PriorityClassName is the name of a priority class that - shall be used for the etcd pods. - type: string - replicas: - format: int32 - type: integer - schedulingConstraints: - description: |- - SchedulingConstraints defines the different scheduling constraints that must be applied to the - pod spec in the etcd statefulset. - Currently supported constraints are Affinity and TopologySpreadConstraints. - properties: - affinity: - description: |- - Affinity defines the various affinity and anti-affinity rules for a pod - that are honoured by the kube-scheduler. - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: + replicas: + format: int32 + type: integer + schedulingConstraints: + description: |- + SchedulingConstraints defines the different scheduling constraints that must be applied to the + pod spec in the etcd statefulset. + Currently supported constraints are Affinity and TopologySpreadConstraints. + properties: + affinity: + description: |- + Affinity defines the various affinity and anti-affinity rules for a pod + that are honoured by the kube-scheduler. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - type: array - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: + type: array + required: - key - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: + type: array + required: - key - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer + type: array + type: object + x-kubernetes-map-type: atomic + type: array required: - - podAffinityTerm - - weight + - nodeSelectorTerms type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: - type: string + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object type: array - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: + x-kubernetes-map-type: atomic + namespaceSelector: description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: - type: string + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object type: array - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object type: object - type: array - matchLabels: - additionalProperties: + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: + type: array + required: - key - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: + type: array + required: - key - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: type: string - required: + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: - type: string + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object type: array - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: + x-kubernetes-map-type: atomic + namespaceSelector: description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: - type: string + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object type: array - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object type: object - type: array - matchLabels: - additionalProperties: + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology domains, - that are honoured by the kube-scheduler. - items: - description: TopologySpreadConstraint specifies how to spread - matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string required: - - key - - operator + - topologyKey type: object type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object type: object - x-kubernetes-map-type: atomic - matchLabelKeys: -<<<<<<< HEAD - description: "MatchLabelKeys is a set of pod label keys - to select the pods over which spreading will be calculated. - The keys are used to lookup values from the incoming pod - labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading - will be calculated for the incoming pod. The same key - is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't - set. Keys that don't exist in the incoming pod labels - will be ignored. A null or empty list means only match - against labelSelector. \n This is a beta field and requires - the MatchLabelKeysInPodTopologySpread feature gate to - be enabled (enabled by default)." -======= - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. ->>>>>>> 8571402f (Fix and run `make generate`) - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - - - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable type: object - type: array - type: object - selector: - description: |- - selector is a label query over pods that should match the replica count. - It must match the pod template's labels. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: + topologySpreadConstraints: description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: + TopologySpreadConstraints describes how a group of pods ought to spread across topology domains, + that are honoured by the kube-scheduler. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string - type: array - required: - - key - - operator + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + selector: + description: |- + selector is a label query over pods that should match the replica count. + It must match the pod template's labels. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - sharedConfig: - description: SharedConfig defines parameters shared and used by Etcd - as well as backup-restore sidecar. - properties: - autoCompactionMode: - description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore - sidecar. - enum: - - periodic - - revision - type: string - autoCompactionRetention: - description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore - sidecar. - type: string - type: object - storageCapacity: - anyOf: - - type: integer - - type: string - description: StorageCapacity defines the size of persistent volume. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - storageClass: - description: |- - StorageClass defines the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeClaimTemplate: - description: VolumeClaimTemplate defines the volume claim template - to be created - type: string - required: - - backup - - etcd - - labels - - replicas - - selector - type: object - status: - description: EtcdStatus defines the observed state of Etcd. - properties: - clusterSize: - description: |- - Cluster size is the current size of the etcd cluster. - Deprecated: this field will not be populated with any value and will be removed in the future. - format: int32 - type: integer - conditions: - description: Conditions represents the latest available observations - of an etcd's current state. - items: - description: Condition holds the information about the state of - a resource. + type: object + x-kubernetes-map-type: atomic + sharedConfig: + description: SharedConfig defines parameters shared and used by Etcd + as well as backup-restore sidecar. properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateTime: - description: Last time the condition was updated. - format: date-time + autoCompactionMode: + description: AutoCompactionMode defines the auto-compaction-mode:'periodic' + mode or 'revision' mode for etcd and embedded-Etcd of backup-restore + sidecar. + enum: + - periodic + - revision type: string - message: - description: A human readable message indicating details about - the transition. + autoCompactionRetention: + description: AutoCompactionRetention defines the auto-compaction-retention + length for etcd as well as for embedded-Etcd of backup-restore + sidecar. type: string - reason: - description: The reason for the condition's last transition. + type: object + storageCapacity: + anyOf: + - type: integer + - type: string + description: StorageCapacity defines the size of persistent volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: |- + StorageClass defines the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeClaimTemplate: + description: VolumeClaimTemplate defines the volume claim template + to be created + type: string + required: + - backup + - etcd + - labels + - replicas + - selector + type: object + status: + description: EtcdStatus defines the observed state of Etcd. + properties: + clusterSize: + description: |- + Cluster size is the current size of the etcd cluster. + Deprecated: this field will not be populated with any value and will be removed in the future. + format: int32 + type: integer + conditions: + description: Conditions represents the latest available observations + of an etcd's current state. + items: + description: Condition holds the information about the state of + a resource. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: Last time the condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of the Etcd condition. + type: string + required: + - lastTransitionTime + - lastUpdateTime + - message + - reason + - status + - type + type: object + type: array + currentReplicas: + description: CurrentReplicas is the current replica count for the + etcd cluster. + format: int32 + type: integer + etcd: + description: CrossVersionObjectReference contains enough information + to let you identify the referred resource. + properties: + apiVersion: + description: API version of the referent type: string - status: - description: Status of the condition, one of True, False, Unknown. + kind: + description: Kind of the referent type: string - type: - description: Type of the Etcd condition. + name: + description: Name of the referent type: string - required: - - lastTransitionTime - - lastUpdateTime - - message - - reason - - status - - type type: object - type: array - currentReplicas: - description: CurrentReplicas is the current replica count for the - etcd cluster. - format: int32 - type: integer - etcd: - description: CrossVersionObjectReference contains enough information - to let you identify the referred resource. - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: Kind of the referent - type: string - name: - description: Name of the referent - type: string - type: object - labelSelector: - description: |- - LabelSelector is a label query over pods that should match the replica count. - It must match the pod template's labels. - Deprecated: this field will be removed in the future. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: + labelSelector: + description: |- + LabelSelector is a label query over pods that should match the replica count. + It must match the pod template's labels. + Deprecated: this field will be removed in the future. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. type: string - type: array - required: - - key - - operator + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + x-kubernetes-map-type: atomic + lastError: + description: |- + LastError represents the last occurred error. + Deprecated: Use LastErrors instead. + type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + description: LastError stores details of the most recent error encountered + for a resource. + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + observedAt: + description: ObservedAt is the time the error was observed. + format: date-time + type: string + required: + - code + - description + - observedAt type: object - type: object - x-kubernetes-map-type: atomic - lastError: - description: |- - LastError represents the last occurred error. - Deprecated: Use LastErrors instead. - type: string - lastErrors: - description: LastErrors captures errors that occurred during the last - operation. - items: - description: LastError stores details of the most recent error encountered - for a resource. + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. properties: - code: - description: Code is an error code that uniquely identifies - an error. - type: string description: - description: Description is a human-readable message indicating - details of the error. - type: string - observedAt: - description: ObservedAt is the time the error was observed. - format: date-time - type: string - required: - - code - - description - - observedAt - type: object - type: array - lastOperation: - description: LastOperation indicates the last operation performed - on this resource. - properties: - description: - description: Description describes the last operation. - type: string - lastUpdateTime: - description: LastUpdateTime is the time at which the operation - was updated. - format: date-time - type: string - runID: - description: |- - RunID correlates an operation with a reconciliation run. - Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is - generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this - as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering - reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. - type: string - state: - description: State is the state of the last operation. - type: string - type: - description: Type is the type of last operation. - type: string - required: - - description - - lastUpdateTime - - runID - - state - - type - type: object - members: - description: Members represents the members of the etcd cluster - items: - description: EtcdMemberStatus holds information about a etcd cluster - membership. - properties: - id: - description: ID is the ID of the etcd member. + description: Description describes the last operation. type: string - lastTransitionTime: - description: LastTransitionTime is the last time the condition's - status changed. + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was updated. format: date-time type: string - name: - description: Name is the name of the etcd member. It is the - name of the backing `Pod`. - type: string - reason: - description: The reason for the condition's last transition. + runID: + description: |- + RunID correlates an operation with a reconciliation run. + Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is + generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this + as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering + reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. type: string - role: - description: Role is the role in the etcd cluster, either `Leader` - or `Member`. + state: + description: State is the state of the last operation. type: string - status: - description: Status of the condition, one of True, False, Unknown. + type: + description: Type is the type of last operation. type: string required: - - lastTransitionTime - - name - - reason - - status + - description + - lastUpdateTime + - runID + - state + - type type: object - type: array - observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this resource. - format: int64 - type: integer - peerUrlTLSEnabled: - description: PeerUrlTLSEnabled captures the state of peer url TLS - being enabled for the etcd member(s) - type: boolean - ready: - description: Ready is `true` if all etcd replicas are ready. - type: boolean - readyReplicas: - description: ReadyReplicas is the count of replicas being ready in - the etcd cluster. - format: int32 - type: integer - replicas: - description: Replicas is the replica count of the etcd resource. - format: int32 - type: integer - serviceName: - description: |- - ServiceName is the name of the etcd service. - Deprecated: this field will be removed in the future. - type: string - updatedReplicas: - description: |- - UpdatedReplicas is the count of updated replicas in the etcd cluster. - Deprecated: this field will be removed in the future. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} + members: + description: Members represents the members of the etcd cluster + items: + description: EtcdMemberStatus holds information about a etcd cluster + membership. + properties: + id: + description: ID is the ID of the etcd member. + type: string + lastTransitionTime: + description: LastTransitionTime is the last time the condition's + status changed. + format: date-time + type: string + name: + description: Name is the name of the etcd member. It is the + name of the backing `Pod`. + type: string + reason: + description: The reason for the condition's last transition. + type: string + role: + description: Role is the role in the etcd cluster, either `Leader` + or `Member`. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + required: + - lastTransitionTime + - name + - reason + - status + type: object + type: array + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this resource. + format: int64 + type: integer + peerUrlTLSEnabled: + description: PeerUrlTLSEnabled captures the state of peer url TLS + being enabled for the etcd member(s) + type: boolean + ready: + description: Ready is `true` if all etcd replicas are ready. + type: boolean + readyReplicas: + description: ReadyReplicas is the count of replicas being ready in + the etcd cluster. + format: int32 + type: integer + replicas: + description: Replicas is the replica count of the etcd resource. + format: int32 + type: integer + serviceName: + description: |- + ServiceName is the name of the etcd service. + Deprecated: this field will be removed in the future. + type: string + updatedReplicas: + description: |- + UpdatedReplicas is the count of updated replicas in the etcd cluster. + Deprecated: this field will be removed in the future. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.labelSelector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: { } \ No newline at end of file diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index c43ae0f30..58c533532 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -253,12 +253,12 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { func (b *stsBuilder) getEtcdContainerVolumeMounts() []corev1.VolumeMount { etcdVolumeMounts := make([]corev1.VolumeMount, 0, 6) etcdVolumeMounts = append(etcdVolumeMounts, b.getEtcdDataVolumeMount()) - etcdVolumeMounts = append(etcdVolumeMounts, b.getSecretVolumeMounts()...) + etcdVolumeMounts = append(etcdVolumeMounts, b.getEtcdSecretVolumeMounts()...) return etcdVolumeMounts } func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMount { - brVolumeMounts := make([]corev1.VolumeMount, 0, 8) + brVolumeMounts := make([]corev1.VolumeMount, 0, 6) brVolumeMounts = append(brVolumeMounts, b.getEtcdDataVolumeMount(), corev1.VolumeMount{ @@ -271,28 +271,30 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun if b.etcd.IsBackupStoreEnabled() { etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() if etcdBackupVolumeMount != nil { - brVolumeMounts = append(brVolumeMounts, *b.getEtcdBackupVolumeMount()) + brVolumeMounts = append(brVolumeMounts, *etcdBackupVolumeMount) } } - return brVolumeMounts } func (b *stsBuilder) getBackupRestoreSecretVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ - { - Name: backRestoreCAVolumeName, - MountPath: backupRestoreCAVolumeMountPath, - }, - { - Name: backRestoreServerTLSVolumeName, - MountPath: backupRestoreServerTLSVolumeMountPath, - }, - { - Name: backRestoreClientTLSVolumeName, - MountPath: backupRestoreClientTLSVolumeMountPath, - }, + if b.etcd.Spec.Backup.TLS != nil { + return []corev1.VolumeMount{ + { + Name: backRestoreCAVolumeName, + MountPath: backupRestoreCAVolumeMountPath, + }, + { + Name: backRestoreServerTLSVolumeName, + MountPath: backupRestoreServerTLSVolumeMountPath, + }, + { + Name: backRestoreClientTLSVolumeName, + MountPath: backupRestoreClientTLSVolumeMountPath, + }, + } } + return []corev1.VolumeMount{} } func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { @@ -301,25 +303,25 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { if b.etcd.Spec.Backup.Store.Container != nil { if b.useEtcdWrapper { return &corev1.VolumeMount{ - Name: "host-storage", + Name: localBackupVolumeName, MountPath: fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, "")), } } else { return &corev1.VolumeMount{ - Name: "host-storage", + Name: localBackupVolumeName, MountPath: pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, ""), } } } case utils.GCS: return &corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/.gcp/", + Name: providerBackupVolumeName, + MountPath: gcsBackupVolumeMountPath, } case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: return &corev1.VolumeMount{ - Name: "etcd-backup", - MountPath: "/var/etcd-backup/", + Name: providerBackupVolumeName, + MountPath: nonGCSProviderBackupVolumeMountPath, } } return nil @@ -342,12 +344,12 @@ func (b *stsBuilder) getEtcdContainer() corev1.Container { ReadinessProbe: b.getEtcdContainerReadinessProbe(), Ports: []corev1.ContainerPort{ { - Name: "server", + Name: serverPortName, Protocol: corev1.ProtocolTCP, ContainerPort: b.serverPort, }, { - Name: "client", + Name: clientPortName, Protocol: corev1.ProtocolTCP, ContainerPort: b.clientPort, }, @@ -370,7 +372,7 @@ func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { Args: b.getBackupRestoreContainerCommandArgs(), Ports: []corev1.ContainerPort{ { - Name: "server", + Name: serverPortName, Protocol: corev1.ProtocolTCP, ContainerPort: b.backupPort, }, @@ -641,27 +643,27 @@ func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { } } -func (b *stsBuilder) getSecretVolumeMounts() []corev1.VolumeMount { +func (b *stsBuilder) getEtcdSecretVolumeMounts() []corev1.VolumeMount { secretVolumeMounts := make([]corev1.VolumeMount, 0, 5) if b.etcd.Spec.Etcd.ClientUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: "client-url-ca-etcd", - MountPath: "/var/etcd/ssl/client/ca", + Name: clientCAVolumeName, + MountPath: etcdCAVolumeMountPath, }, corev1.VolumeMount{ - Name: "client-url-etcd-server-tls", - MountPath: "/var/etcd/ssl/client/server", + Name: serverTLSVolumeName, + MountPath: etcdServerTLSVolumeMountPath, }, corev1.VolumeMount{ - Name: "client-url-etcd-client-tls", - MountPath: "/var/etcd/ssl/client/client", + Name: clientTLSVolumeName, + MountPath: etcdClientTLSVolumeMountPath, }) } if b.etcd.Spec.Etcd.PeerUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: "peer-url-ca-etcd", - MountPath: "/var/etcd/ssl/peer/ca", + Name: peerCAVolumeName, + MountPath: etcdPeerCAVolumeMountPath, }, corev1.VolumeMount{ - Name: "peer-url-etcd-server-tls", - MountPath: "/var/etcd/ssl/peer/server", + Name: peerServerTLSVolumeName, + MountPath: etcdPeerServerTLSVolumeMountPath, }) } return secretVolumeMounts @@ -806,7 +808,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol hpt := corev1.HostPathDirectory return &corev1.Volume{ - Name: "host-storage", + Name: localBackupVolumeName, VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: hostPath + "/" + pointer.StringDeref(store.Container, ""), @@ -820,7 +822,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol } return &corev1.Volume{ - Name: "etcd-backup", + Name: providerBackupVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, diff --git a/internal/operator/statefulset/constants.go b/internal/operator/statefulset/constants.go index f6940d133..7cc2d4a99 100644 --- a/internal/operator/statefulset/constants.go +++ b/internal/operator/statefulset/constants.go @@ -18,6 +18,8 @@ const ( backRestoreServerTLSVolumeName = "back-restore-server-tls" backRestoreClientTLSVolumeName = "back-restore-client-tls" etcdConfigVolumeName = "etcd-config-file" + localBackupVolumeName = "local-backup" + providerBackupVolumeName = "etcd-backup" ) // constants for volume mount paths @@ -25,6 +27,13 @@ const ( backupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" backupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" backupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" + etcdCAVolumeMountPath = "/var/etcd/ssl/ca" + etcdServerTLSVolumeMountPath = "/var/etcd/ssl/server" + etcdClientTLSVolumeMountPath = "/var/etcd/ssl/client" + etcdPeerCAVolumeMountPath = "/var/etcd/ssl/peer/ca" + etcdPeerServerTLSVolumeMountPath = "/var/etcd/ssl/peer/server" + gcsBackupVolumeMountPath = "/var/.gcp/" + nonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" ) const ( @@ -36,3 +45,9 @@ const ( // etcdDataVolumeMountPath is the path on etcd and etcd-backup-restore containers where etcd data directory is hosted. etcdDataVolumeMountPath = "/var/etcd/data" ) + +// constants for container ports +const ( + serverPortName = "server" + clientPortName = "client" +) diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index b1c1e3c3e..ced9fbfd3 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -165,23 +165,19 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { return BeEmpty() } initContainerMatcherElements := make(map[string]gomegatypes.GomegaMatcher) - volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) changePermissionsInitContainerMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal("change-permissions"), + "Name": Equal(common.ChangePermissionsInitContainerName), "Image": Equal(s.initContainerImage), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Command": HaveExactElements("sh", "-c", "--"), "Args": HaveExactElements("chown -R 65532:65532 /var/etcd/data"), - "VolumeMounts": ConsistOf(MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumeClaimTemplateName), - "MountPath": Equal("/var/etcd/data"), - })), + "VolumeMounts": ConsistOf(s.matchEtcdDataVolMount()), "SecurityContext": matchPodInitContainerSecurityContext(), }) - initContainerMatcherElements["change-permissions"] = changePermissionsInitContainerMatcher + initContainerMatcherElements[common.ChangePermissionsInitContainerName] = changePermissionsInitContainerMatcher if s.etcd.IsBackupStoreEnabled() && s.provider != nil && *s.provider == utils.Local { changeBackupBucketPermissionsMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal("change-backup-bucket-permissions"), + "Name": Equal(common.ChangeBackupBucketPermissionsInitContainerName), "Image": Equal(s.initContainerImage), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Command": HaveExactElements("sh", "-c", "--"), @@ -189,7 +185,7 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { "VolumeMounts": s.matchBackupRestoreContainerVolMounts(), "SecurityContext": matchPodInitContainerSecurityContext(), }) - initContainerMatcherElements["change-backup-bucket-permissions"] = changeBackupBucketPermissionsMatcher + initContainerMatcherElements[common.ChangeBackupBucketPermissionsInitContainerName] = changeBackupBucketPermissionsMatcher } return MatchAllElements(containerIdentifier, initContainerMatcherElements) } @@ -216,12 +212,12 @@ func (s StatefulSetMatcher) matchEtcdContainer() gomegatypes.GomegaMatcher { })), "Ports": ConsistOf( MatchFields(IgnoreExtras, Fields{ - "Name": Equal("server"), + "Name": Equal(serverPortName), "Protocol": Equal(corev1.ProtocolTCP), "ContainerPort": Equal(s.serverPort), }), MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client"), + "Name": Equal(clientPortName), "Protocol": Equal(corev1.ProtocolTCP), "ContainerPort": Equal(s.clientPort), }), @@ -235,8 +231,8 @@ func (s StatefulSetMatcher) matchEtcdContainer() gomegatypes.GomegaMatcher { func (s StatefulSetMatcher) matchEtcdContainerVolMounts() gomegatypes.GomegaMatcher { volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) - secretVolMountMatchers := s.getSecretVolMountsMatchers() - if len(secretVolMountMatchers) > 0 { + secretVolMountMatchers := s.getEtcdSecretVolMountsMatchers() + if secretVolMountMatchers != nil && len(secretVolMountMatchers) > 0 { volMountMatchers = append(volMountMatchers, secretVolMountMatchers...) } return ConsistOf(volMountMatchers) @@ -252,7 +248,7 @@ func (s StatefulSetMatcher) matchBackupRestoreContainer() gomegatypes.GomegaMatc //"Args": Equal(s.matchBackupRestoreContainerCmdArgs()), "Ports": ConsistOf( MatchFields(IgnoreExtras, Fields{ - "Name": Equal("server"), + "Name": Equal(serverPortName), "Protocol": Equal(corev1.ProtocolTCP), "ContainerPort": Equal(s.backupPort), }), @@ -329,14 +325,14 @@ func (s StatefulSetMatcher) matchEtcdContainerCmdArgs() gomegatypes.GomegaMatche func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) - return matchVolMount(volumeClaimTemplateName, "/var/etcd/data") + return matchVolMount(volumeClaimTemplateName, etcdDataVolumeMountPath) } func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.GomegaMatcher { - volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 8) + volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) - volMountMatchers = append(volMountMatchers, matchVolMount("etcd-config-file", "/var/etcd/config/")) - volMountMatchers = append(volMountMatchers, s.getSecretVolMountsMatchers()...) + volMountMatchers = append(volMountMatchers, matchVolMount(etcdConfigFileName, etcdConfigFileMountPath)) + volMountMatchers = append(volMountMatchers, s.getBackupRestoreSecretVolMountMatchers()...) if s.etcd.IsBackupStoreEnabled() { etcdBackupVolMountMatcher := s.getEtcdBackupVolumeMountMatcher() if etcdBackupVolMountMatcher != nil { @@ -346,16 +342,26 @@ func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.G return ConsistOf(volMountMatchers) } -func (s StatefulSetMatcher) getSecretVolMountsMatchers() []gomegatypes.GomegaMatcher { +func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatypes.GomegaMatcher { + secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 3) + if s.etcd.Spec.Backup.TLS != nil { + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backRestoreCAVolumeName, backupRestoreCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backRestoreServerTLSVolumeName, backupRestoreServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backRestoreClientTLSVolumeName, backupRestoreClientTLSVolumeMountPath)) + } + return secretVolMountMatchers +} + +func (s StatefulSetMatcher) getEtcdSecretVolMountsMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("client-url-ca-etcd", "/var/etcd/ssl/client/ca")) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("client-url-etcd-server-tls", "/var/etcd/ssl/client/server")) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("client-url-etcd-client-tls", "/var/etcd/ssl/client/client")) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(clientCAVolumeName, etcdCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(serverTLSVolumeName, etcdServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(clientTLSVolumeName, etcdClientTLSVolumeMountPath)) } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("peer-url-ca-etcd", "/var/etcd/ssl/peer/ca")) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount("", "/var/etcd/ssl/peer/server")) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(peerCAVolumeName, etcdPeerCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(peerServerTLSVolumeName, etcdPeerServerTLSVolumeMountPath)) } return secretVolMountMatchers } @@ -365,15 +371,15 @@ func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.Gomega case utils.Local: if s.etcd.Spec.Backup.Store.Container != nil { if s.useEtcdWrapper { - return matchVolMount("host-storage", "/home/nonroot/"+pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) + return matchVolMount(localBackupVolumeName, fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))) } else { - return matchVolMount("host-storage", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) + return matchVolMount(localBackupVolumeName, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) } } case utils.GCS: - return matchVolMount("etcd-backup", "/var/.gcp/") + return matchVolMount(providerBackupVolumeName, gcsBackupVolumeMountPath) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - return matchVolMount("etcd-backup", "/var/etcd-backup/") + return matchVolMount(providerBackupVolumeName, nonGCSProviderBackupVolumeMountPath) } return nil } @@ -409,15 +415,15 @@ func (s StatefulSetMatcher) matchEtcdPodSecurityContext() gomegatypes.GomegaMatc func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { volMatchers := make([]gomegatypes.GomegaMatcher, 0, 7) etcdConfigFileVolMountMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), + "Name": Equal(etcdConfigVolumeName), "VolumeSource": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ "Name": Equal(s.etcd.GetConfigMapName()), }), "Items": HaveExactElements(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ - "Key": Equal("etcd.conf.yaml"), - "Path": Equal("etcd.conf.yaml"), + "Key": Equal(etcdConfigFileName), + "Path": Equal(etcdConfigFileName), })), "DefaultMode": PointTo(Equal(int32(0644))), })), @@ -425,7 +431,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { }) volMatchers = append(volMatchers, etcdConfigFileVolMountMatcher) secretVolMatchers := s.getPodSecurityVolumeMatchers() - if len(secretVolMatchers) > 0 { + if secretVolMatchers != nil && len(secretVolMatchers) > 0 { volMatchers = append(volMatchers, secretVolMatchers...) } if s.etcd.IsBackupStoreEnabled() { @@ -441,7 +447,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM volMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-ca-etcd"), + "Name": Equal(clientCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), @@ -449,7 +455,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-server-tls"), + "Name": Equal(serverTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), @@ -457,7 +463,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal("client-url-etcd-client-tls"), + "Name": Equal(clientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), @@ -467,7 +473,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-ca-etcd"), + "Name": Equal(peerCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), @@ -476,7 +482,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal("peer-url-etcd-server-tls"), + "Name": Equal(peerServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), @@ -484,6 +490,34 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) } + if s.etcd.Spec.Backup.TLS != nil { + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal(backRestoreCAVolumeName), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), + })), + }), + })) + + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal(backRestoreServerTLSVolumeName), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), + })), + }), + })) + + volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ + "Name": Equal(backRestoreClientTLSVolumeName), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), + })), + }), + })) + } return volMatchers } @@ -496,7 +530,7 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { hostPath, err := utils.GetHostMountPathFromSecretRef(context.Background(), s.cl, logr.Discard(), s.etcd.Spec.Backup.Store, s.etcd.Namespace) s.g.Expect(err).ToNot(HaveOccurred()) return MatchFields(IgnoreExtras, Fields{ - "Name": Equal("host-storage"), + "Name": Equal(localBackupVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "HostPath": PointTo(MatchFields(IgnoreExtras, Fields{ "Path": Equal(fmt.Sprintf("%s/%s", hostPath, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))), From 9e59dba0e755c75097abc0f19a3244aeccae8d77 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 19 Mar 2024 11:21:35 +0530 Subject: [PATCH 108/235] small fix in handlePeerTLSChanges and added etcd IT tests to Makefile --- Makefile | 5 +++-- internal/operator/statefulset/statefulset.go | 21 ++++---------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 9988950fb..2d97a8a9c 100644 --- a/Makefile +++ b/Makefile @@ -132,8 +132,9 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) @bash $(HACK_DIR)/e2e-test/run-e2e-test.sh $(PROVIDERS) .PHONY: test-integration -test-integration: $(GINKGO) $(SETUP_ENVTEST) - @bash $(HACK_DIR)/test.sh ./test/integration/... +test-integration: set-permissions $(GINKGO) $(SETUP_ENVTEST) + @"$(REPO_ROOT)/hack/test.sh" ./test/integration/... + @go test -v ./test/it/... .PHONY: update-dependencies update-dependencies: diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 988302110..af00bfe15 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -174,26 +174,13 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru ctx.Logger.Info("Secret volume mounts to enable Peer URL TLS have already been mounted. Skipping patching StatefulSet with secret volume mounts.") } // check again if peer TLS has been enabled for all members. If not then force a requeue of the reconcile request. + peerTLSEnabledForAllMembers, err = utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) + if err != nil { + return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) + } if !peerTLSEnabledForAllMembers { return fmt.Errorf("peer URL TLS not enabled for all members for etcd: %v, requeuing reconcile request", etcd.GetNamespaceName()) } - - //ctx.Logger.Info("Attempting to enable TLS for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) - // - //tlsEnabled, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) - //if err != nil { - // return fmt.Errorf("error verifying if TLS is enabled post-patch: %v", err) - //} - //// It usually takes time for TLS to be enabled and reflected via the lease. So first time around this will not be true. - //// So instead of waiting we requeue the request to be re-tried again. - //if !tlsEnabled { - // return fmt.Errorf("failed to enable TLS for etcd [name: %s, namespace: %s]", etcd.Name, etcd.Namespace) - //} - // - //if err := deleteAllStsPods(ctx, r.client, "Deleting all StatefulSet pods due to TLS enablement", existingSts); err != nil { - // return fmt.Errorf("error deleting StatefulSet pods after enabling TLS: %v", err) - //} - //ctx.Logger.Info("TLS enabled for etcd peers", "namespace", etcd.Namespace, "name", etcd.Name) } ctx.Logger.Info("Peer URL TLS has been enabled for all members") return nil From 31d12ea8aa5cec9d85a7b889c72bd55e82ac09d1 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 21 Mar 2024 10:50:30 +0530 Subject: [PATCH 109/235] Fix statefulset etcd.conf.yaml volumemount name --- internal/operator/statefulset/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 58c533532..b8ed4cf5e 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -262,7 +262,7 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun brVolumeMounts = append(brVolumeMounts, b.getEtcdDataVolumeMount(), corev1.VolumeMount{ - Name: etcdConfigFileName, + Name: etcdConfigVolumeName, MountPath: etcdConfigFileMountPath, }, ) From 422a6a8e5c164484fca47bbb328cb6d4daa7e28f Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 21 Mar 2024 10:51:23 +0530 Subject: [PATCH 110/235] Fix DataVolumesReady condition sts nil check --- internal/health/condition/check_data_volumes_ready.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go index 06091102e..9101775c2 100644 --- a/internal/health/condition/check_data_volumes_ready.go +++ b/internal/health/condition/check_data_volumes_ready.go @@ -27,12 +27,11 @@ func (d *dataVolumesReady) Check(ctx context.Context, etcd druidv1alpha1.Etcd) R } sts, err := utils.GetStatefulSet(ctx, d.cl, &etcd) - if err != nil { - if errors.IsNotFound(err) { - res.reason = "StatefulSetNotFound" - res.message = fmt.Sprintf("StatefulSet %s not found for etcd", etcd.Name) - return res - } + if errors.IsNotFound(err) || sts == nil { + res.reason = "StatefulSetNotFound" + res.message = fmt.Sprintf("StatefulSet %s not found for etcd", etcd.Name) + return res + } else if err != nil { res.reason = "UnableToFetchStatefulSet" res.message = fmt.Sprintf("Unable to fetch StatefulSet for etcd: %s", err.Error()) return res From 79d3f17f09b710e6efb30aaf652cbf4ec3fc24b4 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 21 Mar 2024 10:52:29 +0530 Subject: [PATCH 111/235] Improve sentinel webhook resource decoder logic --- internal/webhook/sentinel/handler.go | 193 ++++++--------------------- 1 file changed, 41 insertions(+), 152 deletions(-) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 564375f98..d48b7c3c5 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -20,10 +20,10 @@ import ( policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -47,54 +47,29 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { Client: mgr.GetClient(), config: config, decoder: decoder, - logger: log.Log.WithName(handlerName), + logger: mgr.GetLogger().WithName(handlerName), }, nil } // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { - var ( - requestGK = schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} - decodeOldObject bool - obj client.Object - err error - ) - - log := h.logger.WithValues("resourceGroup", req.Kind.Group, "resourceKind", req.Kind.Kind, "name", req.Name, "namespace", req.Namespace, "operation", req.Operation, "user", req.UserInfo.Username) + requestGKString := fmt.Sprintf("%s/%s", req.Kind.Group, req.Kind.Kind) + log := h.logger.WithValues("name", req.Name, "namespace", req.Namespace, "resourceGroupKind", requestGKString, "operation", req.Operation, "user", req.UserInfo.Username) log.Info("Sentinel webhook invoked") - if req.Operation == admissionv1.Delete { - decodeOldObject = true + requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} + if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && + req.Operation == admissionv1.Update { + return admission.Allowed("lease resource can be freely updated") } - switch requestGK { - case corev1.SchemeGroupVersion.WithKind("ServiceAccount").GroupKind(): - obj, err = h.decodeServiceAccount(req, decodeOldObject) - case corev1.SchemeGroupVersion.WithKind("Service").GroupKind(): - obj, err = h.decodeService(req, decodeOldObject) - case corev1.SchemeGroupVersion.WithKind("ConfigMap").GroupKind(): - obj, err = h.decodeConfigMap(req, decodeOldObject) - case rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(): - obj, err = h.decodeRole(req, decodeOldObject) - case rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(): - obj, err = h.decodeRoleBinding(req, decodeOldObject) - case appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(): - obj, err = h.decodeStatefulSet(req, decodeOldObject) - case policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(): - obj, err = h.decodePodDisruptionBudget(req, decodeOldObject) - case batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(): - obj, err = h.decodeJob(req, decodeOldObject) - case coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): - if req.Operation == admissionv1.Update { - return admission.Allowed("lease resource can be freely updated") - } - obj, err = h.decodeLease(req, decodeOldObject) - default: - return admission.Allowed(fmt.Sprintf("unexpected resource type: %s", requestGK)) - } + obj, err := h.decodeRequestObject(req, requestGK) if err != nil { return admission.Errored(http.StatusInternalServerError, err) } + if obj == nil { + return admission.Allowed(fmt.Sprintf("unexpected resource type: %s", requestGKString)) + } etcdName, hasLabel := obj.GetLabels()[druidv1alpha1.LabelPartOfKey] if !hasLabel { @@ -137,128 +112,42 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) } -func (h *Handler) decodeServiceAccount(req admission.Request, decodeOldObject bool) (client.Object, error) { - sa := &corev1.ServiceAccount{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, sa); err != nil { - return nil, err - } - return sa, nil - } - if err := h.decoder.Decode(req, sa); err != nil { - return nil, err - } - return sa, nil -} - -func (h *Handler) decodeService(req admission.Request, decodeOldObject bool) (client.Object, error) { - svc := &corev1.Service{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, svc); err != nil { - return nil, err - } - return svc, nil - } - if err := h.decoder.Decode(req, svc); err != nil { - return nil, err - } - return svc, nil -} - -func (h *Handler) decodeConfigMap(req admission.Request, decodeOldObject bool) (client.Object, error) { - cm := &corev1.ConfigMap{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, cm); err != nil { - return nil, err - } - return cm, nil - } - if err := h.decoder.Decode(req, cm); err != nil { - return nil, err - } - return cm, nil -} - -func (h *Handler) decodeRole(req admission.Request, decodeOldObject bool) (client.Object, error) { - role := &rbacv1.Role{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, role); err != nil { - return nil, err - } - return role, nil - } - if err := h.decoder.Decode(req, role); err != nil { - return nil, err - } - return role, nil -} - -func (h *Handler) decodeRoleBinding(req admission.Request, decodeOldObject bool) (client.Object, error) { - rb := &rbacv1.RoleBinding{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, rb); err != nil { - return nil, err - } - return rb, nil - } - if err := h.decoder.Decode(req, rb); err != nil { - return nil, err - } - return rb, nil -} - -func (h *Handler) decodeStatefulSet(req admission.Request, decodeOldObject bool) (client.Object, error) { - sts := &appsv1.StatefulSet{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, sts); err != nil { - return nil, err - } - return sts, nil - } - if err := h.decoder.Decode(req, sts); err != nil { - return nil, err - } - return sts, nil +func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.GroupKind) (client.Object, error) { + var ( + obj client.Object + err error + ) + switch requestGK { + case + corev1.SchemeGroupVersion.WithKind("ServiceAccount").GroupKind(), + corev1.SchemeGroupVersion.WithKind("Service").GroupKind(), + corev1.SchemeGroupVersion.WithKind("ConfigMap").GroupKind(), + rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(), + rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(), + appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), + policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), + batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), + coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): + obj, err = h.doDecodeRequestObject(req) + } + return obj, err } -func (h *Handler) decodePodDisruptionBudget(req admission.Request, decodeOldObject bool) (client.Object, error) { - pdb := &policyv1.PodDisruptionBudget{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, pdb); err != nil { - return nil, err - } - return pdb, nil - } - if err := h.decoder.Decode(req, pdb); err != nil { - return nil, err - } - return pdb, nil -} +func (h *Handler) doDecodeRequestObject(req admission.Request) (client.Object, error) { + var ( + err error + obj = &unstructured.Unstructured{} + ) -func (h *Handler) decodeJob(req admission.Request, decodeOldObject bool) (client.Object, error) { - job := &batchv1.Job{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, job); err != nil { + if req.Operation == admissionv1.Delete { + if err = h.decoder.DecodeRaw(req.OldObject, obj); err != nil { return nil, err } - return job, nil - } - if err := h.decoder.Decode(req, job); err != nil { - return nil, err + return obj, nil } - return job, nil -} -func (h *Handler) decodeLease(req admission.Request, decodeOldObject bool) (client.Object, error) { - lease := &coordinationv1.Lease{} - if decodeOldObject { - if err := h.decoder.DecodeRaw(req.OldObject, lease); err != nil { - return nil, err - } - return lease, nil - } - if err := h.decoder.Decode(req, lease); err != nil { + if err = h.decoder.Decode(req, obj); err != nil { return nil, err } - return lease, nil + return obj, nil } From f885f64a7a71b6a15ca35e6c9ce4f1f14cf1b8d3 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sat, 23 Mar 2024 00:36:40 +0530 Subject: [PATCH 112/235] Add unit tests for FetchPVCWarningMessagesForStatefulSet util func --- .../condition/check_data_volumes_ready.go | 2 +- internal/utils/statefulset.go | 8 +- internal/utils/statefulset_test.go | 93 ++++++++++++++++++- test/utils/client.go | 59 +++++++++++- test/utils/event.go | 29 ++++++ test/utils/pvc.go | 30 ++++++ 6 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 test/utils/event.go create mode 100644 test/utils/pvc.go diff --git a/internal/health/condition/check_data_volumes_ready.go b/internal/health/condition/check_data_volumes_ready.go index 9101775c2..7de03f2ac 100644 --- a/internal/health/condition/check_data_volumes_ready.go +++ b/internal/health/condition/check_data_volumes_ready.go @@ -37,7 +37,7 @@ func (d *dataVolumesReady) Check(ctx context.Context, etcd druidv1alpha1.Etcd) R return res } - pvcEvents, err := utils.FetchPVCWarningMessageForStatefulSet(ctx, d.cl, sts) + pvcEvents, err := utils.FetchPVCWarningMessagesForStatefulSet(ctx, d.cl, sts) if err != nil { res.reason = "UnableToFetchWarningEventsForDataVolumes" res.message = fmt.Sprintf("Unable to fetch warning events for PVCs used by StatefulSet %v: %s", kutil.Key(sts.Name, sts.Namespace), err.Error()) diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index e1799788e..001060160 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -57,12 +57,12 @@ func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.E return nil, nil } -// FetchPVCWarningMessageForStatefulSet fetches warning message for PVCs for a statefulset, if found concatenates the first 2 warning messages and returns +// FetchPVCWarningMessagesForStatefulSet fetches warning messages for PVCs for a statefulset, if found concatenates the first 2 warning messages and returns // them as string warning message. In case it fails to fetch events, it collects the errors and returns the combined error. -func FetchPVCWarningMessageForStatefulSet(ctx context.Context, cl client.Client, sts *appsv1.StatefulSet) (string, error) { +func FetchPVCWarningMessagesForStatefulSet(ctx context.Context, cl client.Client, sts *appsv1.StatefulSet) (string, error) { pvcs := &corev1.PersistentVolumeClaimList{} if err := cl.List(ctx, pvcs, client.InNamespace(sts.GetNamespace())); err != nil { - return "", fmt.Errorf("unable to list PVCs for sts %s: %v", sts.Name, err) + return "", fmt.Errorf("unable to list PVCs for sts %s: %w", sts.Name, err) } var ( @@ -78,7 +78,7 @@ func FetchPVCWarningMessageForStatefulSet(ctx context.Context, cl client.Client, } messages, err := kutil.FetchEventMessages(ctx, cl.Scheme(), cl, &pvc, corev1.EventTypeWarning, 2) if err != nil { - pvcErr = errors.Join(pvcErr, fmt.Errorf("unable to fetch warning events for PVC %s/%s: %v", pvc.Namespace, pvc.Name, err)) + pvcErr = errors.Join(pvcErr, fmt.Errorf("unable to fetch warning events for PVC %s/%s: %w", pvc.Namespace, pvc.Name, err)) } if messages != "" { events = append(events, fmt.Sprintf("Warning for PVC %s/%s: %s", pvc.Namespace, pvc.Name, messages)) diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index 6541b7615..05aad50f2 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -7,18 +7,23 @@ package utils import ( "context" "errors" + "fmt" "testing" testutils "github.com/gardener/etcd-druid/test/utils" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - stsName = "etcd-test-0" stsNamespace = "test-ns" + stsName = "etcd-test" + eventName = "test-event" ) func TestIsStatefulSetReady(t *testing.T) { @@ -173,3 +178,89 @@ func TestGetStatefulSet(t *testing.T) { }) } } + +func TestFetchPVCWarningMessagesForStatefulSet(t *testing.T) { + internalErr := errors.New("test internal error") + apiInternalErr := apierrors.NewInternalError(internalErr) + + etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(3).Build() + sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, etcd.Spec.Replicas) + pvcPending := testutils.CreatePVC(sts, fmt.Sprintf("%s-0", sts.Name), corev1.ClaimPending) + pvcBound := testutils.CreatePVC(sts, fmt.Sprintf("%s-1", sts.Name), corev1.ClaimBound) + eventWarning := testutils.CreateEvent(eventName, sts.Namespace, "FailedMount", "test pvc warning message", corev1.EventTypeWarning, pvcPending, pvcPending.GroupVersionKind()) + + testCases := []struct { + name string + sts *appsv1.StatefulSet + pvcList []client.Object + eventList []client.Object + errors []testutils.ErrorsForGVK + expectedErr *apierrors.StatusError + expectedMsg string + }{ + { + name: "error in listing PVCs", + errors: []testutils.ErrorsForGVK{ + { + GVK: corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaimList"), + ListErr: apiInternalErr, + }, + }, + expectedErr: apiInternalErr, + }, + { + name: "no PVCs found", + pvcList: nil, + expectedErr: nil, + expectedMsg: "", + }, + { + name: "PVCs found but error in listing events", + pvcList: []client.Object{pvcPending}, + errors: []testutils.ErrorsForGVK{ + { + GVK: corev1.SchemeGroupVersion.WithKind("EventList"), + ListErr: apiInternalErr, + }, + }, + expectedErr: apiInternalErr, + expectedMsg: "", + }, + { + name: "PVCs found with warning events", + pvcList: []client.Object{pvcPending}, + eventList: []client.Object{eventWarning}, + expectedErr: nil, + expectedMsg: eventWarning.Message, + }, + { + name: "PVCs found but no warning events", + pvcList: []client.Object{pvcBound}, + expectedMsg: "", + }, + } + + g := NewWithT(t) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var existingObjects []client.Object + if tc.pvcList != nil { + existingObjects = append(existingObjects, tc.pvcList...) + } + if tc.eventList != nil { + existingObjects = append(existingObjects, tc.eventList...) + } + + cl := testutils.CreateTestFakeClientForObjectsInNamespaceWithGVK(tc.errors, etcd.Namespace, existingObjects...) + messages, err := FetchPVCWarningMessagesForStatefulSet(context.Background(), cl, sts) + if tc.expectedErr != nil { + g.Expect(err).To(HaveOccurred()) + g.Expect(errors.Is(err, tc.expectedErr)).To(BeTrue()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(messages).To(Equal(tc.expectedMsg)) + } + }) + } +} diff --git a/test/utils/client.go b/test/utils/client.go index fc675990b..9e633c122 100644 --- a/test/utils/client.go +++ b/test/utils/client.go @@ -11,7 +11,9 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -41,9 +43,16 @@ type errorRecord struct { resourceName string resourceNamespace string labels labels.Set + resourceGVK schema.GroupVersionKind err error } +type ErrorsForGVK struct { + GVK schema.GroupVersionKind + DeleteErr *apierrors.StatusError + ListErr *apierrors.StatusError +} + // TestClientBuilder builds a client.Client which will also react to the configured errors. type TestClientBuilder struct { delegatingClient client.Client @@ -82,6 +91,23 @@ func CreateTestFakeClientForAllObjectsInNamespace(deleteAllErr, listErr *apierro return cl } +// CreateTestFakeClientForObjectsInNamespaceWithGVK is a convenience function which creates a test client which uses a fake client as a delegate and reacts to the configured errors for all objects in the given namespace for the given GroupVersionKinds. +func CreateTestFakeClientForObjectsInNamespaceWithGVK(errors []ErrorsForGVK, namespace string, existingObjects ...client.Object) client.Client { + fakeDelegateClientBuilder := fake.NewClientBuilder() + if existingObjects != nil && len(existingObjects) > 0 { + fakeDelegateClientBuilder.WithObjects(existingObjects...) + } + fakeDelegateClient := fakeDelegateClientBuilder.Build() + + cl := NewTestClientBuilder().WithClient(fakeDelegateClient) + + for _, e := range errors { + cl.RecordErrorForObjectsWithGVK(ClientMethodDeleteAll, namespace, e.GVK, e.DeleteErr). + RecordErrorForObjectsWithGVK(ClientMethodList, namespace, e.GVK, e.ListErr) + } + return cl.Build() +} + // NewTestClientBuilder creates a new instance of TestClientBuilder. func NewTestClientBuilder() *TestClientBuilder { return &TestClientBuilder{} @@ -125,6 +151,21 @@ func (b *TestClientBuilder) RecordErrorForObjectsMatchingLabels(method ClientMet return b } +// RecordErrorForObjectsWithGVK records an error for a specific client.Client method and objects in a given namespace of a given GroupVersionKind. +func (b *TestClientBuilder) RecordErrorForObjectsWithGVK(method ClientMethod, namespace string, gvk schema.GroupVersionKind, err *apierrors.StatusError) *TestClientBuilder { + // this method records error, so if nil error is passed then there is no need to create any error record. + if err == nil { + return b + } + b.errorRecords = append(b.errorRecords, errorRecord{ + method: method, + resourceGVK: gvk, + resourceNamespace: namespace, + err: err, + }) + return b +} + // Build creates a new instance of client.Client which will react to the configured errors. func (b *TestClientBuilder) Build() client.Client { return &testClient{ @@ -151,7 +192,13 @@ func (c *testClient) Get(ctx context.Context, key client.ObjectKey, obj client.O func (c *testClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) - if err := c.getRecordObjectCollectionError(ClientMethodList, listOpts.Namespace, listOpts.LabelSelector); err != nil { + + gvk, err := apiutil.GVKForObject(list, c.delegate.Scheme()) + if err != nil { + return err + } + + if err := c.getRecordedObjectCollectionError(ClientMethodList, listOpts.Namespace, listOpts.LabelSelector, gvk); err != nil { return err } return c.delegate.List(ctx, list, opts...) @@ -174,7 +221,7 @@ func (c *testClient) Delete(ctx context.Context, obj client.Object, opts ...clie func (c *testClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { deleteOpts := client.DeleteAllOfOptions{} deleteOpts.ApplyOptions(opts) - if err := c.getRecordObjectCollectionError(ClientMethodDeleteAll, deleteOpts.Namespace, deleteOpts.LabelSelector); err != nil { + if err := c.getRecordedObjectCollectionError(ClientMethodDeleteAll, deleteOpts.Namespace, deleteOpts.LabelSelector, obj.GetObjectKind().GroupVersionKind()); err != nil { return err } return c.delegate.DeleteAllOf(ctx, obj, opts...) @@ -221,10 +268,12 @@ func (c *testClient) getRecordedObjectError(method ClientMethod, objKey client.O return nil } -func (c *testClient) getRecordObjectCollectionError(method ClientMethod, namespace string, labelSelector labels.Selector) error { +func (c *testClient) getRecordedObjectCollectionError(method ClientMethod, namespace string, labelSelector labels.Selector, objGVK schema.GroupVersionKind) error { for _, errRecord := range c.errorRecords { - if errRecord.method == method && errRecord.resourceNamespace == namespace && labelSelector.Matches(errRecord.labels) { - return errRecord.err + if errRecord.method == method && errRecord.resourceNamespace == namespace { + if errRecord.resourceGVK == objGVK || (labelSelector == nil && errRecord.labels == nil) || labelSelector.Matches(errRecord.labels) { + return errRecord.err + } } } return nil diff --git a/test/utils/event.go b/test/utils/event.go new file mode 100644 index 000000000..0f2ebf67c --- /dev/null +++ b/test/utils/event.go @@ -0,0 +1,29 @@ +package utils + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CreateEvent creates an event with the given parameters. +func CreateEvent(name, namespace, reason, message, eventType string, involvedObject client.Object, involvedObjectGVK schema.GroupVersionKind) *corev1.Event { + return &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + InvolvedObject: corev1.ObjectReference{ + APIVersion: involvedObjectGVK.GroupVersion().String(), + Kind: involvedObjectGVK.Kind, + Name: involvedObject.GetName(), + Namespace: involvedObject.GetNamespace(), + UID: involvedObject.GetUID(), + ResourceVersion: involvedObject.GetResourceVersion(), + }, + Reason: reason, + Message: message, + Type: eventType, + } +} diff --git a/test/utils/pvc.go b/test/utils/pvc.go new file mode 100644 index 000000000..77b196418 --- /dev/null +++ b/test/utils/pvc.go @@ -0,0 +1,30 @@ +package utils + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CreatePVC creates a PVC for the given StatefulSet pod. +func CreatePVC(sts *appsv1.StatefulSet, podName string, phase corev1.PersistentVolumeClaimPhase) *corev1.PersistentVolumeClaim { + if sts == nil { + return nil + } + if sts.Spec.VolumeClaimTemplates == nil || len(sts.Spec.VolumeClaimTemplates) == 0 { + return nil + } + + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", sts.Spec.VolumeClaimTemplates[0].Name, podName), + Namespace: sts.Namespace, + }, + Spec: sts.Spec.VolumeClaimTemplates[0].Spec, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: phase, + }, + } +} From 69a8b09ac68e42fabefb5cc416efcafd0995dd60 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sun, 24 Mar 2024 13:35:22 +0530 Subject: [PATCH 113/235] Fix unit test for FetchPVCWarningMessagesForStatefulSet util func --- internal/utils/event.go | 144 +++++++++++++++++++++++++++++ internal/utils/statefulset.go | 4 +- internal/utils/statefulset_test.go | 6 +- test/utils/event.go | 4 + test/utils/pvc.go | 4 + 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 internal/utils/event.go diff --git a/internal/utils/event.go b/internal/utils/event.go new file mode 100644 index 000000000..9db989476 --- /dev/null +++ b/internal/utils/event.go @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + "fmt" + "strings" + "time" + + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +// fetchEventMessages gets events for the given object of the given `eventType` and returns them as a formatted output. +// The function expects that the given `involvedObj` is specified with a proper `metav1.TypeMeta`. +// This is a temporary re-implementation of the `FetchEventMessages()` function from github.com/gardener/gardener/pkg/utils/kubernetes package, which was required to circumvent the issue of the fake client from controller-runtime v0.14.6 not supporting listing objects by multiple field selectors, which was added in controller-runtime v0.17.0 by https://github.com/kubernetes-sigs/controller-runtime/pull/2512. +// Updating controller-runtime in etcd-druid is not straight-forward due to the dependency on gardener/gardener, which forces the vendor directory to be removed in etcd-druid, which is a large change in itself, and will be addressed by a different PR https://github.com/gardener/etcd-druid/pull/748. +// Once gardener/gardener dependency is updated to v1.90.0+, this function will be removed in favour of the original implementation. +func fetchEventMessages(ctx context.Context, scheme *runtime.Scheme, cl client.Client, involvedObj client.Object, eventType string, eventsLimit int) (string, error) { + events, err := listEvents(ctx, cl, involvedObj.GetNamespace()) + if err != nil { + return "", err + } + + events, err = filterEvents(events, scheme, involvedObj, eventType) + if err != nil { + return "", err + } + + if len(events) > 0 { + return buildEventsErrorMessage(events, eventsLimit), nil + } + return "", nil +} + +// listEvents fetches all events in the given namespace. +func listEvents(ctx context.Context, cl client.Client, namespace string) ([]corev1.Event, error) { + eventList := &corev1.EventList{} + if err := cl.List(ctx, eventList, client.InNamespace(namespace)); err != nil { + return nil, fmt.Errorf("failed to list events: %w", err) + } + + return eventList.Items, nil +} + +// filterEvents filters the given events by the given `eventType` and `involvedObject`. +func filterEvents(events []corev1.Event, scheme *runtime.Scheme, involvedObject client.Object, eventType string) ([]corev1.Event, error) { + gvk, err := apiutil.GVKForObject(involvedObject, scheme) + if err != nil { + return nil, fmt.Errorf("failed to identify GVK for object: %w", err) + } + + apiVersion, kind := gvk.ToAPIVersionAndKind() + if apiVersion == "" { + return nil, fmt.Errorf("apiVersion not specified for object %s/%s", involvedObject.GetNamespace(), involvedObject.GetName()) + } + if kind == "" { + return nil, fmt.Errorf("kind not specified for object %s/%s", involvedObject.GetNamespace(), involvedObject.GetName()) + } + + var filteredEvents []corev1.Event + for _, event := range events { + if event.Type == eventType && + event.InvolvedObject.APIVersion == apiVersion && + event.InvolvedObject.Kind == kind && + event.InvolvedObject.Name == involvedObject.GetName() && + event.InvolvedObject.Namespace == involvedObject.GetNamespace() { + filteredEvents = append(filteredEvents, event) + } + } + + return filteredEvents, nil +} + +func buildEventsErrorMessage(events []corev1.Event, eventsLimit int) string { + sortByLastTimestamp := func(o1, o2 client.Object) bool { + obj1, ok1 := o1.(*corev1.Event) + obj2, ok2 := o2.(*corev1.Event) + + if !ok1 || !ok2 { + return false + } + + return obj1.LastTimestamp.Time.Before(obj2.LastTimestamp.Time) + } + + list := &corev1.EventList{Items: events} + kutil.SortBy(sortByLastTimestamp).Sort(list) + events = list.Items + + if len(events) > eventsLimit { + events = events[len(events)-eventsLimit:] + } + + var builder strings.Builder + fmt.Fprintf(&builder, "-> Events:") + for _, event := range events { + var interval string + if event.Count > 1 { + interval = fmt.Sprintf("%s ago (x%d over %s)", translateTimestampSince(event.LastTimestamp), event.Count, translateTimestampSince(event.FirstTimestamp)) + } else { + interval = fmt.Sprintf("%s ago", translateTimestampSince(event.FirstTimestamp)) + if event.FirstTimestamp.IsZero() { + interval = fmt.Sprintf("%s ago", translateMicroTimestampSince(event.EventTime)) + } + } + source := event.Source.Component + if source == "" { + source = event.ReportingController + } + + fmt.Fprintf(&builder, "\n* %s reported %s: %s", source, interval, event.Message) + } + + return builder.String() +} + +// translateTimestampSince returns the elapsed time since timestamp in +// human-readable approximation. +func translateTimestampSince(timestamp metav1.Time) string { + if timestamp.IsZero() { + return "" + } + + return duration.HumanDuration(time.Since(timestamp.Time)) +} + +// translateMicroTimestampSince returns the elapsed time since timestamp in +// human-readable approximation. +func translateMicroTimestampSince(timestamp metav1.MicroTime) string { + if timestamp.IsZero() { + return "" + } + + return duration.HumanDuration(time.Since(timestamp.Time)) +} diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index 001060160..cbada7d0f 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -12,7 +12,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -76,7 +75,8 @@ func FetchPVCWarningMessagesForStatefulSet(ctx context.Context, cl client.Client if !strings.HasPrefix(pvc.GetName(), pvcPrefix) || pvc.Status.Phase == corev1.ClaimBound { continue } - messages, err := kutil.FetchEventMessages(ctx, cl.Scheme(), cl, &pvc, corev1.EventTypeWarning, 2) + // TODO (shreyas-s-rao): switch to kutil.FetchEventMessages() once g/g is upgraded to v1.90.0+ + messages, err := fetchEventMessages(ctx, cl.Scheme(), cl, &pvc, corev1.EventTypeWarning, 2) if err != nil { pvcErr = errors.Join(pvcErr, fmt.Errorf("unable to fetch warning events for PVC %s/%s: %w", pvc.Namespace, pvc.Name, err)) } diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index 05aad50f2..1448f839a 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "strings" "testing" testutils "github.com/gardener/etcd-druid/test/utils" @@ -16,6 +17,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -187,7 +189,7 @@ func TestFetchPVCWarningMessagesForStatefulSet(t *testing.T) { sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcd.UID, etcd.Spec.Replicas) pvcPending := testutils.CreatePVC(sts, fmt.Sprintf("%s-0", sts.Name), corev1.ClaimPending) pvcBound := testutils.CreatePVC(sts, fmt.Sprintf("%s-1", sts.Name), corev1.ClaimBound) - eventWarning := testutils.CreateEvent(eventName, sts.Namespace, "FailedMount", "test pvc warning message", corev1.EventTypeWarning, pvcPending, pvcPending.GroupVersionKind()) + eventWarning := testutils.CreateEvent(eventName, sts.Namespace, "FailedMount", "test pvc warning message", corev1.EventTypeWarning, pvcPending, schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}) testCases := []struct { name string @@ -259,7 +261,7 @@ func TestFetchPVCWarningMessagesForStatefulSet(t *testing.T) { g.Expect(errors.Is(err, tc.expectedErr)).To(BeTrue()) } else { g.Expect(err).ToNot(HaveOccurred()) - g.Expect(messages).To(Equal(tc.expectedMsg)) + g.Expect(strings.Contains(messages, tc.expectedMsg)).To(BeTrue()) } }) } diff --git a/test/utils/event.go b/test/utils/event.go index 0f2ebf67c..2c67c4579 100644 --- a/test/utils/event.go +++ b/test/utils/event.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/test/utils/pvc.go b/test/utils/pvc.go index 77b196418..42a1d4300 100644 --- a/test/utils/pvc.go +++ b/test/utils/pvc.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( From 9fe7957c1a4c5e4d34d42f00b7fad87cafe5358a Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sun, 24 Mar 2024 13:36:27 +0530 Subject: [PATCH 114/235] Fix errors from `make check` --- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 342 +++++++++--------- .../10-crd-druid.gardener.cloud_etcds.yaml | 4 +- internal/operator/statefulset/stsmatcher.go | 4 +- 3 files changed, 175 insertions(+), 175 deletions(-) diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 87ccdf510..9aa0b085e 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -14,183 +14,183 @@ spec: singular: etcdcopybackupstask scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: EtcdCopyBackupsTask is a task for copying etcd backups from a - source to a target store. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: EtcdCopyBackupsTaskSpec defines the parameters for the copy - backups task. - properties: - maxBackupAge: - description: |- - MaxBackupAge is the maximum age in days that a backup must have in order to be copied. - By default all backups will be copied. - format: int32 - type: integer - maxBackups: - description: MaxBackups is the maximum number of backups that will - be copied starting with the most recent ones. - format: int32 - type: integer - sourceStore: - description: SourceStore defines the specification of the source object - store provider for storing backups. + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: EtcdCopyBackupsTask is a task for copying etcd backups from a + source to a target store. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EtcdCopyBackupsTaskSpec defines the parameters for the copy + backups task. + properties: + maxBackupAge: + description: |- + MaxBackupAge is the maximum age in days that a backup must have in order to be copied. + By default all backups will be copied. + format: int32 + type: integer + maxBackups: + description: MaxBackups is the maximum number of backups that will + be copied starting with the most recent ones. + format: int32 + type: integer + sourceStore: + description: SourceStore defines the specification of the source object + store provider for storing backups. + properties: + container: + description: Container is the name of the container the backup + is stored at. + type: string + prefix: + description: Prefix is the prefix used for the store. + type: string + provider: + description: Provider is the name of the backup provider. + type: string + secretRef: + description: SecretRef is the reference to the secret which used + to connect to the backup store. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - prefix + type: object + targetStore: + description: TargetStore defines the specification of the target object + store provider for storing backups. + properties: + container: + description: Container is the name of the container the backup + is stored at. + type: string + prefix: + description: Prefix is the prefix used for the store. + type: string + provider: + description: Provider is the name of the backup provider. + type: string + secretRef: + description: SecretRef is the reference to the secret which used + to connect to the backup store. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - prefix + type: object + waitForFinalSnapshot: + description: WaitForFinalSnapshot defines the parameters for waiting + for a final full snapshot before copying backups. + properties: + enabled: + description: Enabled specifies whether to wait for a final full + snapshot before copying backups. + type: boolean + timeout: + description: |- + Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups + will be performed anyway. No timeout or 0 means wait forever. + type: string + required: + - enabled + type: object + required: + - sourceStore + - targetStore + type: object + status: + description: EtcdCopyBackupsTaskStatus defines the observed state of the + copy backups task. + properties: + conditions: + description: Conditions represents the latest available observations + of an object's current state. + items: + description: Condition holds the information about the state of + a resource. properties: - container: - description: Container is the name of the container the backup - is stored at. + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time type: string - prefix: - description: Prefix is the prefix used for the store. + lastUpdateTime: + description: Last time the condition was updated. + format: date-time type: string - provider: - description: Provider is the name of the backup provider. + message: + description: A human readable message indicating details about + the transition. type: string - secretRef: - description: SecretRef is the reference to the secret which used - to connect to the backup store. - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which the - secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - prefix - type: object - targetStore: - description: TargetStore defines the specification of the target object - store provider for storing backups. - properties: - container: - description: Container is the name of the container the backup - is stored at. + reason: + description: The reason for the condition's last transition. type: string - prefix: - description: Prefix is the prefix used for the store. + status: + description: Status of the condition, one of True, False, Unknown. type: string - provider: - description: Provider is the name of the backup provider. + type: + description: Type of the Etcd condition. type: string - secretRef: - description: SecretRef is the reference to the secret which used - to connect to the backup store. - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which the - secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic required: - - prefix + - lastTransitionTime + - lastUpdateTime + - message + - reason + - status + - type type: object - waitForFinalSnapshot: - description: WaitForFinalSnapshot defines the parameters for waiting - for a final full snapshot before copying backups. - properties: - enabled: - description: Enabled specifies whether to wait for a final full - snapshot before copying backups. - type: boolean - timeout: - description: |- - Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups - will be performed anyway. No timeout or 0 means wait forever. - type: string - required: - - enabled - type: object - required: - - sourceStore - - targetStore - type: object - status: - description: EtcdCopyBackupsTaskStatus defines the observed state of the - copy backups task. - properties: - conditions: - description: Conditions represents the latest available observations - of an object's current state. - items: - description: Condition holds the information about the state of - a resource. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateTime: - description: Last time the condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of the Etcd condition. - type: string - required: - - lastTransitionTime - - lastUpdateTime - - message - - reason - - status - - type - type: object - type: array - lastError: - description: LastError represents the last occurred error. - type: string - observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this resource. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} \ No newline at end of file + type: array + lastError: + description: LastError represents the last occurred error. + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this resource. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index fdbea37a9..140f7b07e 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: etcds.druid.gardener.cloud spec: group: druid.gardener.cloud @@ -1904,4 +1904,4 @@ spec: labelSelectorPath: .status.labelSelector specReplicasPath: .spec.replicas statusReplicasPath: .status.replicas - status: { } \ No newline at end of file + status: {} \ No newline at end of file diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index ced9fbfd3..4cc1f13fd 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -232,7 +232,7 @@ func (s StatefulSetMatcher) matchEtcdContainerVolMounts() gomegatypes.GomegaMatc volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) secretVolMountMatchers := s.getEtcdSecretVolMountsMatchers() - if secretVolMountMatchers != nil && len(secretVolMountMatchers) > 0 { + if len(secretVolMountMatchers) > 0 { volMountMatchers = append(volMountMatchers, secretVolMountMatchers...) } return ConsistOf(volMountMatchers) @@ -431,7 +431,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { }) volMatchers = append(volMatchers, etcdConfigFileVolMountMatcher) secretVolMatchers := s.getPodSecurityVolumeMatchers() - if secretVolMatchers != nil && len(secretVolMatchers) > 0 { + if len(secretVolMatchers) > 0 { volMatchers = append(volMatchers, secretVolMatchers...) } if s.etcd.IsBackupStoreEnabled() { From fe41c6784a5eb81f9e9009b3b5b1259bbf482cd3 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 25 Mar 2024 11:22:05 +0530 Subject: [PATCH 115/235] Add unit tests for sentinel webhook --- internal/webhook/sentinel/handler.go | 4 + internal/webhook/sentinel/handler_test.go | 512 ++++++++++++++++++++++ test/utils/client.go | 17 + test/utils/etcd.go | 13 + 4 files changed, 546 insertions(+) create mode 100644 internal/webhook/sentinel/handler_test.go diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index d48b7c3c5..e46900f4b 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -63,6 +63,10 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Allowed("lease resource can be freely updated") } + if req.Operation != admissionv1.Update && req.Operation != admissionv1.Delete { + return admission.Allowed(fmt.Sprintf("operation is not %s or %s", admissionv1.Update, admissionv1.Delete)) + } + obj, err := h.decodeRequestObject(req, requestGK) if err != nil { return admission.Errored(http.StatusInternalServerError, err) diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go new file mode 100644 index 000000000..f2c81d029 --- /dev/null +++ b/internal/webhook/sentinel/handler_test.go @@ -0,0 +1,512 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package sentinel + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "testing" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/client/kubernetes" + testutils "github.com/gardener/etcd-druid/test/utils" + + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + admissionv1 "k8s.io/api/admission/v1" + appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var ( + objectsForUpdate = []runtime.Object{ + &corev1.ServiceAccount{}, + &corev1.Service{}, + &corev1.ConfigMap{}, + &rbacv1.Role{}, + &rbacv1.RoleBinding{}, + &appsv1.StatefulSet{}, + &policyv1.PodDisruptionBudget{}, + &batchv1.Job{}, + } + objectsForDelete = append( + objectsForUpdate, + &coordinationv1.Lease{}, + ) +) + +type testCase struct { + name string + // ----- handler configuration ----- + reconcilerServiceAccount string + exemptServiceAccounts []string + // ----- request ----- + userName string + operation admissionv1.Operation + objectKind *schema.GroupVersionKind + objectName string + objectNamespace string + objectLabels map[string]string + object *runtime.RawExtension + oldObject *runtime.RawExtension + // ----- etcd configuration ----- + etcdName string + etcdNamespace string + etcdAnnotations map[string]string + etcdStatusLastOperation *druidv1alpha1.LastOperation + etcdGetErr *apierrors.StatusError + // ----- expected ----- + expectedAllowed bool + expectedReason string + expectedMessage string + expectedCode int32 +} + +// ------------------------ Handle ------------------------ +func TestHandle(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + var ( + reconcilerServiceAccount = "etcd-druid-sa" + exemptServiceAccounts = []string{"exempt-sa-1"} + testUserName = "test-user" + testObjectName = "test" + testNamespace = "test-ns" + testEtcdName = "test" + + internalErr = errors.New("test internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) + apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") + ) + + lease := &coordinationv1.Lease{} + leaseJSON, err := json.Marshal(lease) + g.Expect(err).ToNot(HaveOccurred()) + + deployment := &appsv1.Deployment{} + deploymentJSON, err := json.Marshal(deployment) + g.Expect(err).ToNot(HaveOccurred()) + deploymentGVK, err := apiutil.GVKForObject(deployment, kubernetes.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + + commonTestCases := []testCase{ + { + name: "create operation", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + operation: admissionv1.Create, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("operation is not %s or %s", admissionv1.Update, admissionv1.Delete), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "empty request object", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + object: &runtime.RawExtension{Raw: []byte{}}, + oldObject: &runtime.RawExtension{Raw: []byte{}}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: false, + expectedReason: "", + expectedMessage: "there is no content to decode", + expectedCode: http.StatusInternalServerError, + }, + { + name: "malformed request object", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + object: &runtime.RawExtension{Raw: []byte("foo")}, + oldObject: &runtime.RawExtension{Raw: []byte("foo")}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: false, + expectedReason: "", + expectedMessage: "invalid character", + expectedCode: http.StatusInternalServerError, + }, + { + name: "resource has no part-of label", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "etcd not found", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: apiNotFoundErr, + expectedAllowed: true, + expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "error in getting etcd", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: apiInternalErr, + expectedAllowed: false, + expectedReason: "", + expectedMessage: internalErr.Error(), + expectedCode: http.StatusInternalServerError, + }, + { + name: "etcd reconciliation suspended", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "resource protection annotation set to false", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "etcd is currently being reconciled by druid", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + etcdGetErr: nil, + expectedAllowed: false, + expectedReason: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), + expectedMessage: "", + expectedCode: http.StatusForbidden, + }, + { + name: "etcd is currently being reconciled by druid, but request is from druid", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: reconcilerServiceAccount, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "etcd is not currently being reconciled by druid, and request is from exempt service account", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: exemptServiceAccounts[0], + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "etcd is not currently being reconciled by druid, and request is from non-exempt service account", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: false, + expectedReason: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedMessage: "", + expectedCode: http.StatusForbidden, + }, + } + + for _, operation := range []admissionv1.Operation{admissionv1.Update, admissionv1.Delete} { + objects := objectsForUpdate + if operation == admissionv1.Delete { + objects = objectsForDelete + } + + for _, object := range objects { + for _, tc := range commonTestCases { + var ( + obj, oldObj *unstructured.Unstructured + ) + + oldObj = &unstructured.Unstructured{} + apiVersion, kind := object.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind() + oldObj.SetAPIVersion(apiVersion) + oldObj.SetKind(kind) + oldObj.SetLabels(tc.objectLabels) + oldObj.SetName(tc.objectName) + oldObj.SetNamespace(tc.objectNamespace) + + if operation == admissionv1.Update { + obj = oldObj.DeepCopy() + } + + if tc.object == nil { + rawExt, err := getRawExtensionFromUnstructured(obj) + g.Expect(err).ToNot(HaveOccurred()) + tc.object = &rawExt + } + + if tc.oldObject == nil { + rawExt, err := getRawExtensionFromUnstructured(oldObj) + g.Expect(err).ToNot(HaveOccurred()) + tc.oldObject = &rawExt + } + + if tc.objectKind == nil { + gvk, err := apiutil.GVKForObject(object, kubernetes.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + tc.objectKind = &gvk + } + + if tc.operation == "" { + tc.operation = operation + } + + gvk, err := apiutil.GVKForObject(object, kubernetes.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + + t.Run(fmt.Sprintf("%s for %s operation on object GVK %s/%s/%s", tc.name, tc.operation, gvk.Group, gvk.Version, gvk.Kind), func(t *testing.T) { + resp, err := runTestCase(tc) + g.Expect(err).ToNot(HaveOccurred()) + assertTestResult(g, tc, resp) + }) + } + } + } + + specialTestCases := []testCase{ + { + name: "lease update", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + operation: admissionv1.Update, + objectKind: &schema.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + object: &runtime.RawExtension{Object: lease, Raw: leaseJSON}, + oldObject: &runtime.RawExtension{Object: lease, Raw: leaseJSON}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: "lease resource can be freely updated", + expectedMessage: "", + expectedCode: http.StatusOK, + }, + { + name: "unknown resource type", + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, + operation: admissionv1.Update, + objectKind: &schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + objectName: testObjectName, + objectNamespace: testNamespace, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + object: &runtime.RawExtension{Object: deployment, Raw: deploymentJSON}, + oldObject: &runtime.RawExtension{Object: deployment, Raw: deploymentJSON}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: nil, + etcdStatusLastOperation: nil, + etcdGetErr: nil, + expectedAllowed: true, + expectedReason: fmt.Sprintf("unexpected resource type: %s/%s", deploymentGVK.Group, deploymentGVK.Kind), + expectedMessage: "", + expectedCode: http.StatusOK, + }, + } + + for _, tc := range specialTestCases { + t.Run(tc.name, func(t *testing.T) { + resp, err := runTestCase(tc) + g.Expect(err).ToNot(HaveOccurred()) + assertTestResult(g, tc, resp) + }) + } +} + +func runTestCase(tc testCase) (*admission.Response, error) { + etcd := testutils.EtcdBuilderWithDefaults(tc.etcdName, tc.etcdNamespace). + WithAnnotations(tc.etcdAnnotations). + WithLastOperation(tc.etcdStatusLastOperation). + Build() + existingObjects := []client.Object{etcd} + + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, existingObjects, client.ObjectKey{Name: tc.etcdName, Namespace: tc.etcdNamespace}) + + config := &Config{ + Enabled: true, + ReconcilerServiceAccount: tc.reconcilerServiceAccount, + ExemptServiceAccounts: tc.exemptServiceAccounts, + } + + decoder, err := admission.NewDecoder(cl.Scheme()) + if err != nil { + return nil, err + } + + handler := &Handler{ + Client: cl, + config: config, + decoder: decoder, + logger: logr.Discard(), + } + + resp := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: tc.operation, + Kind: metav1.GroupVersionKind{Group: tc.objectKind.Group, Version: tc.objectKind.Version, Kind: tc.objectKind.Kind}, + Name: tc.objectName, + Namespace: tc.objectNamespace, + UserInfo: authenticationv1.UserInfo{Username: tc.userName}, + Object: *tc.object, + OldObject: *tc.oldObject, + }, + }) + + return &resp, nil +} + +func getRawExtensionFromUnstructured(obj *unstructured.Unstructured) (runtime.RawExtension, error) { + if obj == nil { + return runtime.RawExtension{}, nil + } + + ro := runtime.Object(obj) + objJSON, err := json.Marshal(ro) + if err != nil { + return runtime.RawExtension{}, err + } + return runtime.RawExtension{ + Object: ro, + Raw: objJSON, + }, nil +} + +func assertTestResult(g *WithT, tc testCase, resp *admission.Response) { + g.Expect(resp.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedReason)) + g.Expect(resp.Result.Message).To(ContainSubstring(tc.expectedMessage)) + g.Expect(resp.Result.Code).To(Equal(tc.expectedCode)) +} diff --git a/test/utils/client.go b/test/utils/client.go index 9e633c122..24b10ce91 100644 --- a/test/utils/client.go +++ b/test/utils/client.go @@ -76,6 +76,23 @@ func CreateTestFakeClientForObjects(getErr, createErr, patchErr, deleteErr *apie return testClientBuilder.Build() } +// CreateTestFakeClientWithSchemeForObjects is a convenience function which creates a test client which uses a fake client with the given scheme as a delegate and reacts to the configured errors for the given object keys. +func CreateTestFakeClientWithSchemeForObjects(scheme *runtime.Scheme, getErr, createErr, patchErr, deleteErr *apierrors.StatusError, existingObjects []client.Object, objKeys ...client.ObjectKey) client.Client { + fakeDelegateClientBuilder := fake.NewClientBuilder().WithScheme(scheme) + if existingObjects != nil && len(existingObjects) > 0 { + fakeDelegateClientBuilder.WithObjects(existingObjects...) + } + fakeDelegateClient := fakeDelegateClientBuilder.Build() + testClientBuilder := NewTestClientBuilder().WithClient(fakeDelegateClient) + for _, objKey := range objKeys { + testClientBuilder.RecordErrorForObjects(ClientMethodGet, getErr, objKey). + RecordErrorForObjects(ClientMethodCreate, createErr, objKey). + RecordErrorForObjects(ClientMethodDelete, deleteErr, objKey). + RecordErrorForObjects(ClientMethodPatch, patchErr, objKey) + } + return testClientBuilder.Build() +} + // CreateTestFakeClientForAllObjectsInNamespace is a convenience function which creates a test client which uses a fake client as a delegate and reacts to the configured errors for all objects in the given namespace matching the given labels. func CreateTestFakeClientForAllObjectsInNamespace(deleteAllErr, listErr *apierrors.StatusError, namespace string, matchingLabels map[string]string, existingObjects ...client.Object) client.Client { fakeDelegateClientBuilder := fake.NewClientBuilder() diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 2edfbafc2..197325d7c 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -141,11 +141,24 @@ func (eb *EtcdBuilder) WithReadyStatus() *EtcdBuilder { Conditions: []druidv1alpha1.Condition{ {Type: druidv1alpha1.ConditionTypeAllMembersReady, Status: druidv1alpha1.ConditionTrue}, }, + LastErrors: []druidv1alpha1.LastError{}, + LastOperation: &druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: druidv1alpha1.LastOperationStateSucceeded, + Description: "Reconciliation succeeded", + RunID: "12345", + LastUpdateTime: metav1.Now(), + }, } return eb } +func (eb *EtcdBuilder) WithLastOperation(operation *druidv1alpha1.LastOperation) *EtcdBuilder { + eb.etcd.Status.LastOperation = operation + return eb +} + func (eb *EtcdBuilder) WithStorageProvider(provider druidv1alpha1.StorageProvider) *EtcdBuilder { // TODO: there is no default case right now which is not very right, returning an error in a default case makes it difficult to chain // This should be improved later From f426672257bb348583cef41aa2df4c37c59cba20 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 25 Mar 2024 13:11:02 +0530 Subject: [PATCH 116/235] Fix unit test for statefulset matcher --- internal/operator/statefulset/stsmatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 4cc1f13fd..7241621a5 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -331,7 +331,7 @@ func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.GomegaMatcher { volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) - volMountMatchers = append(volMountMatchers, matchVolMount(etcdConfigFileName, etcdConfigFileMountPath)) + volMountMatchers = append(volMountMatchers, matchVolMount(etcdConfigVolumeName, etcdConfigFileMountPath)) volMountMatchers = append(volMountMatchers, s.getBackupRestoreSecretVolMountMatchers()...) if s.etcd.IsBackupStoreEnabled() { etcdBackupVolMountMatcher := s.getEtcdBackupVolumeMountMatcher() From d8622c12a5a9cc49c209159a2c68e7931665afd0 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 25 Mar 2024 18:57:19 +0530 Subject: [PATCH 117/235] Clean up unit tests for sentinel webhook --- internal/webhook/sentinel/handler_test.go | 307 +++++++--------------- 1 file changed, 92 insertions(+), 215 deletions(-) diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index f2c81d029..c1a727ef5 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -98,236 +98,126 @@ func TestHandle(t *testing.T) { apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") ) - lease := &coordinationv1.Lease{} - leaseJSON, err := json.Marshal(lease) - g.Expect(err).ToNot(HaveOccurred()) - - deployment := &appsv1.Deployment{} - deploymentJSON, err := json.Marshal(deployment) - g.Expect(err).ToNot(HaveOccurred()) - deploymentGVK, err := apiutil.GVKForObject(deployment, kubernetes.Scheme) + deploymentGVK, err := apiutil.GVKForObject(&appsv1.Deployment{}, kubernetes.Scheme) g.Expect(err).ToNot(HaveOccurred()) commonTestCases := []testCase{ { - name: "create operation", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - operation: admissionv1.Create, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: true, - expectedReason: fmt.Sprintf("operation is not %s or %s", admissionv1.Update, admissionv1.Delete), - expectedMessage: "", - expectedCode: http.StatusOK, + name: "create operation", + operation: admissionv1.Create, + expectedAllowed: true, + expectedReason: fmt.Sprintf("operation is not %s or %s", admissionv1.Update, admissionv1.Delete), + expectedCode: http.StatusOK, }, { - name: "empty request object", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - object: &runtime.RawExtension{Raw: []byte{}}, - oldObject: &runtime.RawExtension{Raw: []byte{}}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: false, - expectedReason: "", - expectedMessage: "there is no content to decode", - expectedCode: http.StatusInternalServerError, + name: "empty request object", + object: &runtime.RawExtension{Raw: []byte{}}, + oldObject: &runtime.RawExtension{Raw: []byte{}}, + expectedAllowed: false, + expectedMessage: "there is no content to decode", + expectedCode: http.StatusInternalServerError, }, { - name: "malformed request object", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - object: &runtime.RawExtension{Raw: []byte("foo")}, - oldObject: &runtime.RawExtension{Raw: []byte("foo")}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: false, - expectedReason: "", - expectedMessage: "invalid character", - expectedCode: http.StatusInternalServerError, + name: "malformed request object", + object: &runtime.RawExtension{Raw: []byte("foo")}, + oldObject: &runtime.RawExtension{Raw: []byte("foo")}, + expectedAllowed: false, + expectedMessage: "invalid character", + expectedCode: http.StatusInternalServerError, }, { - name: "resource has no part-of label", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: true, - expectedReason: fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), - expectedMessage: "", - expectedCode: http.StatusOK, + name: "resource has no part-of label", + objectLabels: map[string]string{}, + expectedAllowed: true, + expectedReason: fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), + expectedCode: http.StatusOK, }, { - name: "etcd not found", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: apiNotFoundErr, - expectedAllowed: true, - expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), - expectedMessage: "", - expectedCode: http.StatusOK, + name: "etcd not found", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdGetErr: apiNotFoundErr, + expectedAllowed: true, + expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), + expectedCode: http.StatusOK, }, { - name: "error in getting etcd", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: apiInternalErr, - expectedAllowed: false, - expectedReason: "", - expectedMessage: internalErr.Error(), - expectedCode: http.StatusInternalServerError, + name: "error in getting etcd", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdGetErr: apiInternalErr, + expectedAllowed: false, + expectedMessage: internalErr.Error(), + expectedCode: http.StatusInternalServerError, }, { - name: "etcd reconciliation suspended", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: true, - expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), - expectedMessage: "", - expectedCode: http.StatusOK, + name: "etcd reconciliation suspended", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, + expectedAllowed: true, + expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), + expectedCode: http.StatusOK, }, { - name: "resource protection annotation set to false", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: true, - expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), - expectedMessage: "", - expectedCode: http.StatusOK, + name: "resource protection annotation set to false", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdName: testEtcdName, + etcdNamespace: testNamespace, + etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, + expectedAllowed: true, + expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedCode: http.StatusOK, }, { name: "etcd is currently being reconciled by druid", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdName: testEtcdName, etcdNamespace: testNamespace, - etcdAnnotations: nil, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, - etcdGetErr: nil, + reconcilerServiceAccount: reconcilerServiceAccount, + userName: testUserName, expectedAllowed: false, expectedReason: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), - expectedMessage: "", expectedCode: http.StatusForbidden, }, { name: "etcd is currently being reconciled by druid, but request is from druid", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: reconcilerServiceAccount, - objectName: testObjectName, - objectNamespace: testNamespace, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdName: testEtcdName, etcdNamespace: testNamespace, - etcdAnnotations: nil, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, - etcdGetErr: nil, + reconcilerServiceAccount: reconcilerServiceAccount, + userName: reconcilerServiceAccount, expectedAllowed: true, expectedReason: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), - expectedMessage: "", expectedCode: http.StatusOK, }, { name: "etcd is not currently being reconciled by druid, and request is from exempt service account", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: exemptServiceAccounts[0], - objectName: testObjectName, - objectNamespace: testNamespace, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdName: testEtcdName, etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: exemptServiceAccounts[0], expectedAllowed: true, expectedReason: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), - expectedMessage: "", expectedCode: http.StatusOK, }, { name: "etcd is not currently being reconciled by druid, and request is from non-exempt service account", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - objectName: testObjectName, - objectNamespace: testNamespace, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdName: testEtcdName, etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + userName: testUserName, expectedAllowed: false, expectedReason: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), - expectedMessage: "", expectedCode: http.StatusForbidden, }, } @@ -344,6 +234,13 @@ func TestHandle(t *testing.T) { obj, oldObj *unstructured.Unstructured ) + if tc.objectName == "" { + tc.objectName = testObjectName + } + if tc.objectNamespace == "" { + tc.objectNamespace = testNamespace + } + oldObj = &unstructured.Unstructured{} apiVersion, kind := object.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind() oldObj.SetAPIVersion(apiVersion) @@ -392,52 +289,32 @@ func TestHandle(t *testing.T) { specialTestCases := []testCase{ { - name: "lease update", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - operation: admissionv1.Update, - objectKind: &schema.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - object: &runtime.RawExtension{Object: lease, Raw: leaseJSON}, - oldObject: &runtime.RawExtension{Object: lease, Raw: leaseJSON}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: true, - expectedReason: "lease resource can be freely updated", - expectedMessage: "", - expectedCode: http.StatusOK, + name: "lease update", + operation: admissionv1.Update, + objectKind: &schema.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, + expectedAllowed: true, + expectedReason: "lease resource can be freely updated", + expectedCode: http.StatusOK, }, { - name: "unknown resource type", - reconcilerServiceAccount: reconcilerServiceAccount, - exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, - operation: admissionv1.Update, - objectKind: &schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, - objectName: testObjectName, - objectNamespace: testNamespace, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - object: &runtime.RawExtension{Object: deployment, Raw: deploymentJSON}, - oldObject: &runtime.RawExtension{Object: deployment, Raw: deploymentJSON}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, - etcdAnnotations: nil, - etcdStatusLastOperation: nil, - etcdGetErr: nil, - expectedAllowed: true, - expectedReason: fmt.Sprintf("unexpected resource type: %s/%s", deploymentGVK.Group, deploymentGVK.Kind), - expectedMessage: "", - expectedCode: http.StatusOK, + name: "unknown resource type", + operation: admissionv1.Update, + objectKind: &schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + expectedAllowed: true, + expectedReason: fmt.Sprintf("unexpected resource type: %s/%s", deploymentGVK.Group, deploymentGVK.Kind), + expectedCode: http.StatusOK, }, } for _, tc := range specialTestCases { + if tc.object == nil { + rawExt, err := getRawExtensionFromUnstructured(&unstructured.Unstructured{}) + g.Expect(err).ToNot(HaveOccurred()) + tc.object = &rawExt + } + if tc.oldObject == nil { + tc.oldObject = tc.object + } t.Run(tc.name, func(t *testing.T) { resp, err := runTestCase(tc) g.Expect(err).ToNot(HaveOccurred()) From 0d3df8e1a917337d5fe3da4cb423afbe25fb4f89 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 27 Mar 2024 12:11:21 +0530 Subject: [PATCH 118/235] minor change for handling create/connect operation --- internal/webhook/sentinel/handler.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index e46900f4b..da87781b9 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "net/http" + "slices" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -28,6 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) +var allowedOperations = []admissionv1.Operation{admissionv1.Create, admissionv1.Connect} + // Handler is the Sentinel Webhook admission handler. type Handler struct { client.Client @@ -53,6 +56,11 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { + + if slices.Contains(allowedOperations, req.Operation) { + return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) + } + requestGKString := fmt.Sprintf("%s/%s", req.Kind.Group, req.Kind.Kind) log := h.logger.WithValues("name", req.Name, "namespace", req.Namespace, "resourceGroupKind", requestGKString, "operation", req.Operation, "user", req.UserInfo.Username) log.Info("Sentinel webhook invoked") @@ -63,10 +71,6 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Allowed("lease resource can be freely updated") } - if req.Operation != admissionv1.Update && req.Operation != admissionv1.Delete { - return admission.Allowed(fmt.Sprintf("operation is not %s or %s", admissionv1.Update, admissionv1.Delete)) - } - obj, err := h.decodeRequestObject(req, requestGK) if err != nil { return admission.Errored(http.StatusInternalServerError, err) From 7a34ba2c06a6da8151a564e1babed3afc354d849 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 27 Mar 2024 18:35:54 +0530 Subject: [PATCH 119/235] Streamline sentinel webhook unit tests --- .../operator/memberlease/memberlease_test.go | 2 - internal/webhook/sentinel/handler.go | 1 - internal/webhook/sentinel/handler_test.go | 566 +++++++++++------- 3 files changed, 339 insertions(+), 230 deletions(-) diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index aa1ba8915..e220067c5 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -253,8 +253,6 @@ func TestTriggerDelete(t *testing.T) { // ***************** Setup operator and test ***************** operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - memberLeasesBeforeDelete := getLatestMemberLeases(g, cl, etcd) - fmt.Println(memberLeasesBeforeDelete) err := operator.TriggerDelete(opCtx, etcd) memberLeasesPostDelete := getLatestMemberLeases(g, cl, etcd) if tc.expectedErr != nil { diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index da87781b9..15667bdb7 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -56,7 +56,6 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { - if slices.Contains(allowedOperations, req.Operation) { return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) } diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index c1a727ef5..46cf50d98 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -21,11 +21,6 @@ import ( admissionv1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -33,57 +28,115 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -var ( - objectsForUpdate = []runtime.Object{ - &corev1.ServiceAccount{}, - &corev1.Service{}, - &corev1.ConfigMap{}, - &rbacv1.Role{}, - &rbacv1.RoleBinding{}, - &appsv1.StatefulSet{}, - &policyv1.PodDisruptionBudget{}, - &batchv1.Job{}, +func getObjectGVK(g *WithT, obj runtime.Object) schema.GroupVersionKind { + gvk, err := apiutil.GVKForObject(obj, kubernetes.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + return gvk +} + +func buildObject(gvk schema.GroupVersionKind, name, namespace string, labels map[string]string) runtime.Object { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetName(name) + obj.SetNamespace(namespace) + obj.SetLabels(labels) + return obj +} + +func TestHandleCreate(t *testing.T) { + g := NewWithT(t) + + testCases := []struct { + name string + expectedAllowed bool + expectedReason string + }{ + { + name: "create any resource", + expectedAllowed: true, + expectedReason: "operation CREATE is allowed", + }, } - objectsForDelete = append( - objectsForUpdate, - &coordinationv1.Lease{}, - ) -) -type testCase struct { - name string - // ----- handler configuration ----- - reconcilerServiceAccount string - exemptServiceAccounts []string - // ----- request ----- - userName string - operation admissionv1.Operation - objectKind *schema.GroupVersionKind - objectName string - objectNamespace string - objectLabels map[string]string - object *runtime.RawExtension - oldObject *runtime.RawExtension - // ----- etcd configuration ----- - etcdName string - etcdNamespace string - etcdAnnotations map[string]string - etcdStatusLastOperation *druidv1alpha1.LastOperation - etcdGetErr *apierrors.StatusError - // ----- expected ----- - expectedAllowed bool - expectedReason string - expectedMessage string - expectedCode int32 + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := fake.NewClientBuilder().Build() + decoder, err := admission.NewDecoder(cl.Scheme()) + if err != nil { + g.Expect(err).ToNot(HaveOccurred()) + } + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), + } + + resp := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + }, + }) + + g.Expect(resp.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedReason)) + }) + } } -// ------------------------ Handle ------------------------ -func TestHandle(t *testing.T) { +func TestHandleLeaseUpdate(t *testing.T) { + g := NewWithT(t) + + testCases := []struct { + name string + gvk metav1.GroupVersionKind + expectedAllowed bool + expectedReason string + }{ + { + name: "update Lease", + expectedAllowed: true, + expectedReason: "lease resource can be freely updated", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := fake.NewClientBuilder().Build() + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), + } + + resp := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Kind: metav1.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, + }, + }) + + g.Expect(resp.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedReason)) + }) + } +} + +func TestHandleUpdate(t *testing.T) { g := NewWithT(t) - t.Parallel() var ( reconcilerServiceAccount = "etcd-druid-sa" @@ -98,29 +151,35 @@ func TestHandle(t *testing.T) { apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") ) - deploymentGVK, err := apiutil.GVKForObject(&appsv1.Deployment{}, kubernetes.Scheme) - g.Expect(err).ToNot(HaveOccurred()) - - commonTestCases := []testCase{ - { - name: "create operation", - operation: admissionv1.Create, - expectedAllowed: true, - expectedReason: fmt.Sprintf("operation is not %s or %s", admissionv1.Update, admissionv1.Delete), - expectedCode: http.StatusOK, - }, + testCases := []struct { + name string + // ----- request ----- + userName string + objectLabels map[string]string + objectRaw []byte + // ----- etcd configuration ----- + etcdAnnotations map[string]string + etcdStatusLastOperation *druidv1alpha1.LastOperation + etcdGetErr *apierrors.StatusError + // ----- handler configuration ----- + reconcilerServiceAccount string + exemptServiceAccounts []string + // ----- expected ----- + expectedAllowed bool + expectedReason string + expectedMessage string + expectedCode int32 + }{ { name: "empty request object", - object: &runtime.RawExtension{Raw: []byte{}}, - oldObject: &runtime.RawExtension{Raw: []byte{}}, + objectRaw: []byte{}, expectedAllowed: false, expectedMessage: "there is no content to decode", expectedCode: http.StatusInternalServerError, }, { name: "malformed request object", - object: &runtime.RawExtension{Raw: []byte("foo")}, - oldObject: &runtime.RawExtension{Raw: []byte("foo")}, + objectRaw: []byte("foo"), expectedAllowed: false, expectedMessage: "invalid character", expectedCode: http.StatusInternalServerError, @@ -135,8 +194,6 @@ func TestHandle(t *testing.T) { { name: "etcd not found", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, etcdGetErr: apiNotFoundErr, expectedAllowed: true, expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), @@ -145,8 +202,6 @@ func TestHandle(t *testing.T) { { name: "error in getting etcd", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, etcdGetErr: apiInternalErr, expectedAllowed: false, expectedMessage: internalErr.Error(), @@ -155,8 +210,6 @@ func TestHandle(t *testing.T) { { name: "etcd reconciliation suspended", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, expectedAllowed: true, expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), @@ -165,8 +218,6 @@ func TestHandle(t *testing.T) { { name: "resource protection annotation set to false", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, expectedAllowed: true, expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), @@ -174,216 +225,277 @@ func TestHandle(t *testing.T) { }, { name: "etcd is currently being reconciled by druid", + userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, - userName: testUserName, expectedAllowed: false, expectedReason: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, { name: "etcd is currently being reconciled by druid, but request is from druid", + userName: reconcilerServiceAccount, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, - userName: reconcilerServiceAccount, expectedAllowed: true, expectedReason: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), expectedCode: http.StatusOK, }, { name: "etcd is not currently being reconciled by druid, and request is from exempt service account", + userName: exemptServiceAccounts[0], objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, - userName: exemptServiceAccounts[0], expectedAllowed: true, expectedReason: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { name: "etcd is not currently being reconciled by druid, and request is from non-exempt service account", + userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdName: testEtcdName, - etcdNamespace: testNamespace, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, - userName: testUserName, expectedAllowed: false, expectedReason: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, } - for _, operation := range []admissionv1.Operation{admissionv1.Update, admissionv1.Delete} { - objects := objectsForUpdate - if operation == admissionv1.Delete { - objects = objectsForDelete - } - - for _, object := range objects { - for _, tc := range commonTestCases { - var ( - obj, oldObj *unstructured.Unstructured - ) - - if tc.objectName == "" { - tc.objectName = testObjectName - } - if tc.objectNamespace == "" { - tc.objectNamespace = testNamespace - } - - oldObj = &unstructured.Unstructured{} - apiVersion, kind := object.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind() - oldObj.SetAPIVersion(apiVersion) - oldObj.SetKind(kind) - oldObj.SetLabels(tc.objectLabels) - oldObj.SetName(tc.objectName) - oldObj.SetNamespace(tc.objectNamespace) - - if operation == admissionv1.Update { - obj = oldObj.DeepCopy() - } - - if tc.object == nil { - rawExt, err := getRawExtensionFromUnstructured(obj) - g.Expect(err).ToNot(HaveOccurred()) - tc.object = &rawExt - } - - if tc.oldObject == nil { - rawExt, err := getRawExtensionFromUnstructured(oldObj) - g.Expect(err).ToNot(HaveOccurred()) - tc.oldObject = &rawExt - } - - if tc.objectKind == nil { - gvk, err := apiutil.GVKForObject(object, kubernetes.Scheme) - g.Expect(err).ToNot(HaveOccurred()) - tc.objectKind = &gvk - } - - if tc.operation == "" { - tc.operation = operation - } - - gvk, err := apiutil.GVKForObject(object, kubernetes.Scheme) - g.Expect(err).ToNot(HaveOccurred()) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace). + WithAnnotations(tc.etcdAnnotations). + WithLastOperation(tc.etcdStatusLastOperation). + Build() + + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }, + decoder: decoder, + logger: logr.Discard(), + } - t.Run(fmt.Sprintf("%s for %s operation on object GVK %s/%s/%s", tc.name, tc.operation, gvk.Group, gvk.Version, gvk.Kind), func(t *testing.T) { - resp, err := runTestCase(tc) - g.Expect(err).ToNot(HaveOccurred()) - assertTestResult(g, tc, resp) - }) + sts := &appsv1.StatefulSet{} + obj := runtime.RawExtension{ + Object: buildObject(getObjectGVK(g, sts), testObjectName, testNamespace, tc.objectLabels), + Raw: tc.objectRaw, } - } + if tc.objectRaw == nil { + objRaw, err := json.Marshal(obj.Object) + g.Expect(err).ToNot(HaveOccurred()) + obj.Raw = objRaw + } + + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + UserInfo: authenticationv1.UserInfo{Username: tc.userName}, + Kind: metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, + Name: testObjectName, + Namespace: testNamespace, + Object: obj, + OldObject: obj, + }, + }) + + g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(string(response.Result.Reason)).To(Equal(tc.expectedReason)) + g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) + g.Expect(response.Result.Code).To(Equal(tc.expectedCode)) + }) } +} + +func TestHandleDelete(t *testing.T) { + g := NewWithT(t) - specialTestCases := []testCase{ + var ( + reconcilerServiceAccount = "etcd-druid-sa" + exemptServiceAccounts = []string{"exempt-sa-1"} + testUserName = "test-user" + testObjectName = "test" + testNamespace = "test-ns" + testEtcdName = "test" + + internalErr = errors.New("test internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) + apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") + ) + + testCases := []struct { + name string + // ----- request ----- + userName string + objectLabels map[string]string + objectRaw []byte + // ----- etcd configuration ----- + etcdAnnotations map[string]string + etcdStatusLastOperation *druidv1alpha1.LastOperation + etcdGetErr *apierrors.StatusError + // ----- handler configuration ----- + reconcilerServiceAccount string + exemptServiceAccounts []string + // ----- expected ----- + expectedAllowed bool + expectedReason string + expectedMessage string + expectedCode int32 + }{ { - name: "lease update", - operation: admissionv1.Update, - objectKind: &schema.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, + name: "empty request object", + objectRaw: []byte{}, + expectedAllowed: false, + expectedMessage: "there is no content to decode", + expectedCode: http.StatusInternalServerError, + }, + { + name: "malformed request object", + objectRaw: []byte("foo"), + expectedAllowed: false, + expectedMessage: "invalid character", + expectedCode: http.StatusInternalServerError, + }, + { + name: "resource has no part-of label", + objectLabels: map[string]string{}, expectedAllowed: true, - expectedReason: "lease resource can be freely updated", + expectedReason: fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), + expectedCode: http.StatusOK, + }, + { + name: "etcd not found", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdGetErr: apiNotFoundErr, + expectedAllowed: true, + expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), expectedCode: http.StatusOK, }, { - name: "unknown resource type", - operation: admissionv1.Update, - objectKind: &schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + name: "error in getting etcd", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdGetErr: apiInternalErr, + expectedAllowed: false, + expectedMessage: internalErr.Error(), + expectedCode: http.StatusInternalServerError, + }, + { + name: "etcd reconciliation suspended", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, expectedAllowed: true, - expectedReason: fmt.Sprintf("unexpected resource type: %s/%s", deploymentGVK.Group, deploymentGVK.Kind), + expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), expectedCode: http.StatusOK, }, + { + name: "resource protection annotation set to false", + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, + expectedAllowed: true, + expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedCode: http.StatusOK, + }, + { + name: "etcd is currently being reconciled by druid", + userName: testUserName, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + reconcilerServiceAccount: reconcilerServiceAccount, + expectedAllowed: false, + expectedReason: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), + expectedCode: http.StatusForbidden, + }, + { + name: "etcd is currently being reconciled by druid, but request is from druid", + userName: reconcilerServiceAccount, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + reconcilerServiceAccount: reconcilerServiceAccount, + expectedAllowed: true, + expectedReason: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), + expectedCode: http.StatusOK, + }, + { + name: "etcd is not currently being reconciled by druid, and request is from exempt service account", + userName: exemptServiceAccounts[0], + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + expectedAllowed: true, + expectedReason: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedCode: http.StatusOK, + }, + { + name: "etcd is not currently being reconciled by druid, and request is from non-exempt service account", + userName: testUserName, + objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + expectedAllowed: false, + expectedReason: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedCode: http.StatusForbidden, + }, } - for _, tc := range specialTestCases { - if tc.object == nil { - rawExt, err := getRawExtensionFromUnstructured(&unstructured.Unstructured{}) - g.Expect(err).ToNot(HaveOccurred()) - tc.object = &rawExt - } - if tc.oldObject == nil { - tc.oldObject = tc.object - } + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - resp, err := runTestCase(tc) - g.Expect(err).ToNot(HaveOccurred()) - assertTestResult(g, tc, resp) - }) - } -} - -func runTestCase(tc testCase) (*admission.Response, error) { - etcd := testutils.EtcdBuilderWithDefaults(tc.etcdName, tc.etcdNamespace). - WithAnnotations(tc.etcdAnnotations). - WithLastOperation(tc.etcdStatusLastOperation). - Build() - existingObjects := []client.Object{etcd} - - cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, existingObjects, client.ObjectKey{Name: tc.etcdName, Namespace: tc.etcdNamespace}) - - config := &Config{ - Enabled: true, - ReconcilerServiceAccount: tc.reconcilerServiceAccount, - ExemptServiceAccounts: tc.exemptServiceAccounts, - } - - decoder, err := admission.NewDecoder(cl.Scheme()) - if err != nil { - return nil, err - } - - handler := &Handler{ - Client: cl, - config: config, - decoder: decoder, - logger: logr.Discard(), - } + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace). + WithAnnotations(tc.etcdAnnotations). + WithLastOperation(tc.etcdStatusLastOperation). + Build() - resp := handler.Handle(context.Background(), admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: tc.operation, - Kind: metav1.GroupVersionKind{Group: tc.objectKind.Group, Version: tc.objectKind.Version, Kind: tc.objectKind.Kind}, - Name: tc.objectName, - Namespace: tc.objectNamespace, - UserInfo: authenticationv1.UserInfo{Username: tc.userName}, - Object: *tc.object, - OldObject: *tc.oldObject, - }, - }) + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) - return &resp, nil -} + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }, + decoder: decoder, + logger: logr.Discard(), + } -func getRawExtensionFromUnstructured(obj *unstructured.Unstructured) (runtime.RawExtension, error) { - if obj == nil { - return runtime.RawExtension{}, nil - } + sts := &appsv1.StatefulSet{} + obj := runtime.RawExtension{ + Object: buildObject(getObjectGVK(g, sts), testObjectName, testNamespace, tc.objectLabels), + Raw: tc.objectRaw, + } + if tc.objectRaw == nil { + objRaw, err := json.Marshal(obj.Object) + g.Expect(err).ToNot(HaveOccurred()) + obj.Raw = objRaw + } - ro := runtime.Object(obj) - objJSON, err := json.Marshal(ro) - if err != nil { - return runtime.RawExtension{}, err + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + UserInfo: authenticationv1.UserInfo{Username: tc.userName}, + Kind: metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, + Name: testObjectName, + Namespace: testNamespace, + OldObject: obj, + }, + }) + + g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(string(response.Result.Reason)).To(Equal(tc.expectedReason)) + g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) + g.Expect(response.Result.Code).To(Equal(tc.expectedCode)) + }) } - return runtime.RawExtension{ - Object: ro, - Raw: objJSON, - }, nil -} - -func assertTestResult(g *WithT, tc testCase, resp *admission.Response) { - g.Expect(resp.Allowed).To(Equal(tc.expectedAllowed)) - g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedReason)) - g.Expect(resp.Result.Message).To(ContainSubstring(tc.expectedMessage)) - g.Expect(resp.Result.Code).To(Equal(tc.expectedCode)) } From b49ebe18882f0bd993e1a0a6984a94b26778738b Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 27 Mar 2024 19:36:58 +0530 Subject: [PATCH 120/235] Set mounted files DefaultMode to 640 (changes incorporated from https://github.com/gardener/etcd-druid/pull/772 by @AleksandarSavchev) --- internal/controller/compaction/reconciler.go | 3 +- .../etcdcopybackupstask/reconciler.go | 4 ++- .../etcdcopybackupstask/reconciler_test.go | 6 ++-- internal/operator/statefulset/builder.go | 30 ++++++++++++------- internal/operator/statefulset/stsmatcher.go | 30 ++++++++++++------- test/e2e/utils.go | 12 ++++---- .../controllers/compaction/reconciler_test.go | 15 ++++++---- .../etcdcopybackupstask/reconciler_test.go | 3 +- 8 files changed, 67 insertions(+), 36 deletions(-) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 846fdfb2a..6b3553587 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -434,7 +434,8 @@ func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr. Name: "etcd-backup", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, + SecretName: storeValues.SecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }) diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 20b3f3063..840f810ee 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -374,6 +374,7 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et RunAsGroup: pointer.Int64(65532), RunAsNonRoot: pointer.Bool(true), RunAsUser: pointer.Int64(65532), + FSGroup: pointer.Int64(65532), } } @@ -461,7 +462,8 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a Name: getVolumeNamePrefix(prefix) + "etcd-backup", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: store.SecretRef.Name, + SecretName: store.SecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }) diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index f0d3b8bf2..d81115263 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -611,7 +611,8 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Expect(volumeSource).NotTo(BeNil()) Expect(volumeSource.Secret).NotTo(BeNil()) Expect(*volumeSource.Secret).To(Equal(corev1.SecretVolumeSource{ - SecretName: store.SecretRef.Name, + SecretName: store.SecretRef.Name, + DefaultMode: pointer.Int32(0640), })) }) @@ -924,7 +925,8 @@ func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Ele "Name": Equal(volumePrefix + "etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(store.SecretRef.Name), + "SecretName": Equal(store.SecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), }), diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index b8ed4cf5e..d5a967795 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -640,6 +640,7 @@ func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { RunAsGroup: pointer.Int64(65532), RunAsNonRoot: pointer.Bool(true), RunAsUser: pointer.Int64(65532), + FSGroup: pointer.Int64(65532), } } @@ -685,7 +686,7 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu Path: etcdConfigFileName, }, }, - DefaultMode: pointer.Int32(0644), + DefaultMode: pointer.Int32(0640), }, }, }, @@ -719,7 +720,8 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { Name: clientCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: clientTLSConfig.TLSCASecretRef.Name, + SecretName: clientTLSConfig.TLSCASecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -727,7 +729,8 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { Name: serverTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: clientTLSConfig.ServerTLSSecretRef.Name, + SecretName: clientTLSConfig.ServerTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -735,7 +738,8 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { Name: clientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: clientTLSConfig.ClientTLSSecretRef.Name, + SecretName: clientTLSConfig.ClientTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -749,7 +753,8 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { Name: peerCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: peerTLSConfig.TLSCASecretRef.Name, + SecretName: peerTLSConfig.TLSCASecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -757,7 +762,8 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { Name: peerServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: peerTLSConfig.ServerTLSSecretRef.Name, + SecretName: peerTLSConfig.ServerTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -771,7 +777,8 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { Name: backRestoreCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tlsConfig.TLSCASecretRef.Name, + SecretName: tlsConfig.TLSCASecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -779,7 +786,8 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { Name: backRestoreServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tlsConfig.ServerTLSSecretRef.Name, + SecretName: tlsConfig.ServerTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -787,7 +795,8 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { Name: backRestoreClientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tlsConfig.ClientTLSSecretRef.Name, + SecretName: tlsConfig.ClientTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -825,7 +834,8 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol Name: providerBackupVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: store.SecretRef.Name, + SecretName: store.SecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, nil diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 7241621a5..d6246bf70 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -409,6 +409,7 @@ func (s StatefulSetMatcher) matchEtcdPodSecurityContext() gomegatypes.GomegaMatc "RunAsGroup": PointTo(Equal(int64(65532))), "RunAsNonRoot": PointTo(Equal(true)), "RunAsUser": PointTo(Equal(int64(65532))), + "FSGroup": PointTo(Equal(int64(65532))), })) } @@ -425,7 +426,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { "Key": Equal(etcdConfigFileName), "Path": Equal(etcdConfigFileName), })), - "DefaultMode": PointTo(Equal(int32(0644))), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), }) @@ -450,7 +451,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(clientCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -458,7 +460,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(serverTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -466,7 +469,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(clientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -476,7 +480,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(peerCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -485,7 +490,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(peerServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -495,7 +501,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(backRestoreCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -504,7 +511,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(backRestoreServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -513,7 +521,8 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "Name": Equal(backRestoreClientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), })) @@ -545,7 +554,8 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(s.etcd.Spec.Backup.Store.SecretRef.Name), + "SecretName": Equal(s.etcd.Spec.Backup.Store.SecretRef.Name), + "DefaultMode": PointTo(Equal(int32(0640))), })), }), }) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 7adb3f97b..ab58a4b31 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -841,7 +841,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(420), + DefaultMode: pointer.Int32(416), }, }, }, @@ -850,7 +850,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(420), + DefaultMode: pointer.Int32(416), }, }, }, @@ -859,7 +859,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(420), + DefaultMode: pointer.Int32(416), }, }, }, @@ -973,7 +973,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(420), + DefaultMode: pointer.Int32(416), }, }, }, @@ -982,7 +982,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(420), + DefaultMode: pointer.Int32(416), }, }, }, @@ -991,7 +991,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(420), + DefaultMode: pointer.Int32(416), }, }, }, diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index f0679a142..72dbc6362 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -439,7 +439,8 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "DefaultMode": Equal(pointer.Int32(0640)), })), }), }), @@ -495,7 +496,8 @@ func validateStoreAWSForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "DefaultMode": Equal(pointer.Int32(0640)), })), }), }), @@ -551,7 +553,8 @@ func validateStoreAzureForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1 "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "DefaultMode": Equal(pointer.Int32(0640)), })), }), }), @@ -607,7 +610,8 @@ func validateStoreOpenstackForCompactionJob(instance *druidv1alpha1.Etcd, j *bat "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "DefaultMode": Equal(pointer.Int32(0640)), })), }), }), @@ -664,7 +668,8 @@ func validateStoreAlicloudForCompactionJob(instance *druidv1alpha1.Etcd, j *batc "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), + "DefaultMode": Equal(pointer.Int32(0640)), })), }), }), diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index 404dda2c3..a20e161ab 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -350,7 +350,8 @@ func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Ele "Name": Equal(volumePrefix + "etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(store.SecretRef.Name), + "SecretName": Equal(store.SecretRef.Name), + "DefaultMode": Equal(pointer.Int32(0640)), })), }), }), From a01161202a4057ab17981eb1cc649ac2b0837611 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 28 Mar 2024 21:47:36 +0530 Subject: [PATCH 121/235] Fix integration tests --- Makefile | 3 +++ .../etcdcopybackupstask/reconciler_test.go | 10 ++++++++++ .../etcdcopybackupstask/reconciler_test.go | 12 +++++++++--- .../controllers/secret/reconciler_test.go | 8 ++++---- test/it/controller/etcd/assertions.go | 2 +- test/it/controller/etcd/reconciler_test.go | 4 ++-- test/it/setup/setup.go | 1 + test/utils/client.go | 16 +++++++++++----- 8 files changed, 41 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 2d97a8a9c..d331c634d 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,9 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) .PHONY: test-integration test-integration: set-permissions $(GINKGO) $(SETUP_ENVTEST) @"$(REPO_ROOT)/hack/test.sh" ./test/integration/... + @export KUBEBUILDER_ASSETS="$(${SETUP_ENVTEST} --arch=amd64 use --use-env -p path 1.22)" + @echo "using envtest tools installed at '${KUBEBUILDER_ASSETS}'" + @export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=2m @go test -v ./test/it/... .PHONY: update-dependencies diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index d81115263..ccb8108f2 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -715,6 +715,12 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), + "Labels": MatchKeys(IgnoreExtras, Keys{ + druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelPartOfKey: Equal(task.Name), + druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), + druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), + }), "OwnerReferences": MatchAllElements(testutils.OwnerRefIterator, Elements{ task.Name: MatchAllFields(Fields{ "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), @@ -730,6 +736,10 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Template": MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Labels": MatchKeys(IgnoreExtras, Keys{ + druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelPartOfKey: Equal(task.Name), + druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), + druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), "networking.gardener.cloud/to-dns": Equal("allowed"), "networking.gardener.cloud/to-public-networks": Equal("allowed"), }), diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index a20e161ab..c049e497c 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -135,9 +135,11 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), - "Annotations": MatchKeys(IgnoreExtras, Keys{ - "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", task.Namespace, task.Name)), - "gardener.cloud/owner-type": Equal("etcdcopybackupstask"), + "Labels": MatchKeys(IgnoreExtras, Keys{ + druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelPartOfKey: Equal(task.Name), + druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), + druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), }), "OwnerReferences": MatchAllElements(testutils.OwnerRefIterator, Elements{ task.Name: MatchAllFields(Fields{ @@ -154,6 +156,10 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Template": MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Labels": MatchKeys(IgnoreExtras, Keys{ + druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelPartOfKey: Equal(task.Name), + druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), + druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), "networking.gardener.cloud/to-dns": Equal("allowed"), "networking.gardener.cloud/to-public-networks": Equal("allowed"), }), diff --git a/test/integration/controllers/secret/reconciler_test.go b/test/integration/controllers/secret/reconciler_test.go index 0d4b946da..6158b56a9 100644 --- a/test/integration/controllers/secret/reconciler_test.go +++ b/test/integration/controllers/secret/reconciler_test.go @@ -53,8 +53,8 @@ var _ = Describe("Secret Controller", func() { "peer-url-etcd-server-tls", "etcd-backup", } - errs := testutils.CreateSecrets(ctx, k8sClient, namespace, secretNames...) - Expect(errs).To(BeEmpty()) + err := testutils.CreateSecrets(ctx, k8sClient, namespace, secretNames...) + Expect(err).ToNot(HaveOccurred()) Expect(k8sClient.Create(ctx, etcd)).To(Succeed()) @@ -72,8 +72,8 @@ var _ = Describe("Secret Controller", func() { "peer-url-etcd-server-tls2", "etcd-backup2", } - errs = testutils.CreateSecrets(ctx, k8sClient, namespace, newSecretNames...) - Expect(errs).To(BeEmpty()) + err = testutils.CreateSecrets(ctx, k8sClient, namespace, newSecretNames...) + Expect(err).ToNot(HaveOccurred()) patch := client.MergeFrom(etcd.DeepCopy()) etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name += "2" diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index 81d224c06..1e8b575a5 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -249,7 +249,7 @@ func assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t *testing.T, cl cl slices.Sort(actualErrorCodes) if !slices.Equal(expectedLastErrorCodes, actualErrorCodes) { - return fmt.Errorf("expected lastErrors to be %v, found %v", expectedLastErrorCodes, etcdInstance.Status.LastErrors) + return fmt.Errorf("expected lastErrors to be %v, found %v", expectedLastErrorCodes, actualErrorCodes) } return nil } diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 67a193475..6e04575db 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -20,6 +20,7 @@ import ( "github.com/gardener/etcd-druid/test/it/controller/assets" "github.com/gardener/etcd-druid/test/it/setup" testutils "github.com/gardener/etcd-druid/test/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" . "github.com/onsi/gomega" @@ -29,11 +30,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/featuregate" + testclock "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - - testclock "k8s.io/utils/clock/testing" ) const testNamespacePrefix = "etcd-reconciler-test-" diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index f873d4950..b9a1a47a4 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -10,6 +10,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" diff --git a/test/utils/client.go b/test/utils/client.go index 24b10ce91..996b3ad49 100644 --- a/test/utils/client.go +++ b/test/utils/client.go @@ -48,9 +48,9 @@ type errorRecord struct { } type ErrorsForGVK struct { - GVK schema.GroupVersionKind - DeleteErr *apierrors.StatusError - ListErr *apierrors.StatusError + GVK schema.GroupVersionKind + DeleteAllErr *apierrors.StatusError + ListErr *apierrors.StatusError } // TestClientBuilder builds a client.Client which will also react to the configured errors. @@ -119,7 +119,7 @@ func CreateTestFakeClientForObjectsInNamespaceWithGVK(errors []ErrorsForGVK, nam cl := NewTestClientBuilder().WithClient(fakeDelegateClient) for _, e := range errors { - cl.RecordErrorForObjectsWithGVK(ClientMethodDeleteAll, namespace, e.GVK, e.DeleteErr). + cl.RecordErrorForObjectsWithGVK(ClientMethodDeleteAll, namespace, e.GVK, e.DeleteAllErr). RecordErrorForObjectsWithGVK(ClientMethodList, namespace, e.GVK, e.ListErr) } return cl.Build() @@ -238,7 +238,13 @@ func (c *testClient) Delete(ctx context.Context, obj client.Object, opts ...clie func (c *testClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { deleteOpts := client.DeleteAllOfOptions{} deleteOpts.ApplyOptions(opts) - if err := c.getRecordedObjectCollectionError(ClientMethodDeleteAll, deleteOpts.Namespace, deleteOpts.LabelSelector, obj.GetObjectKind().GroupVersionKind()); err != nil { + + gvk, err := apiutil.GVKForObject(obj, c.delegate.Scheme()) + if err != nil { + return err + } + + if err := c.getRecordedObjectCollectionError(ClientMethodDeleteAll, deleteOpts.Namespace, deleteOpts.LabelSelector, gvk); err != nil { return err } return c.delegate.DeleteAllOf(ctx, obj, opts...) From d41324401a7031f5130bd77f4c5b5faad4845ceb Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sun, 31 Mar 2024 17:50:10 +0530 Subject: [PATCH 122/235] refactored sentinel webhook unit test --- .ci/hack/component_descriptor | 0 .ci/hack/prepare_release | 0 .ci/hack/set_dependency_version | 0 internal/webhook/sentinel/handler.go | 16 +- internal/webhook/sentinel/handler_test.go | 507 +++++++++++++--------- test/utils/client.go | 5 + 6 files changed, 317 insertions(+), 211 deletions(-) mode change 100755 => 100644 .ci/hack/component_descriptor mode change 100755 => 100644 .ci/hack/prepare_release mode change 100755 => 100644 .ci/hack/set_dependency_version diff --git a/.ci/hack/component_descriptor b/.ci/hack/component_descriptor old mode 100755 new mode 100644 diff --git a/.ci/hack/prepare_release b/.ci/hack/prepare_release old mode 100755 new mode 100644 diff --git a/.ci/hack/set_dependency_version b/.ci/hack/set_dependency_version old mode 100755 new mode 100644 diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 15667bdb7..e49398463 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -56,14 +56,15 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { - if slices.Contains(allowedOperations, req.Operation) { - return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) - } - requestGKString := fmt.Sprintf("%s/%s", req.Kind.Group, req.Kind.Kind) log := h.logger.WithValues("name", req.Name, "namespace", req.Namespace, "resourceGroupKind", requestGKString, "operation", req.Operation, "user", req.UserInfo.Username) log.Info("Sentinel webhook invoked") + if slices.Contains(allowedOperations, req.Operation) { + return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) + } + + // Leases (member and snapshot) will be periodically updated by etcd members. Allow updates to leases. requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && req.Operation == admissionv1.Update { @@ -84,18 +85,13 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R } etcd := &druidv1alpha1.Etcd{} - if err := h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { + if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { if apierrors.IsNotFound(err) { return admission.Allowed(fmt.Sprintf("corresponding etcd %s not found", etcdName)) } return admission.Errored(http.StatusInternalServerError, err) } - // allow changes to resources if etcd spec reconciliation is currently suspended - if etcd.IsReconciliationSuspended() { - return admission.Allowed(fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", etcd.Name)) - } - // allow changes to resources if etcd has annotation druid.gardener.cloud/resource-protection: false if !etcd.AreManagedResourcesProtected() { return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index 46cf50d98..b51d31355 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -32,124 +32,160 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -func getObjectGVK(g *WithT, obj runtime.Object) schema.GroupVersionKind { - gvk, err := apiutil.GVKForObject(obj, kubernetes.Scheme) - g.Expect(err).ToNot(HaveOccurred()) - return gvk -} +const ( + testUserName = "test-user" + testObjectName = "test" + testNamespace = "test-ns" + testEtcdName = "test" +) -func buildObject(gvk schema.GroupVersionKind, name, namespace string, labels map[string]string) runtime.Object { - obj := &unstructured.Unstructured{} - obj.SetGroupVersionKind(gvk) - obj.SetName(name) - obj.SetNamespace(namespace) - obj.SetLabels(labels) - return obj -} +var ( + internalErr = errors.New("test internal error") + apiInternalErr = apierrors.NewInternalError(internalErr) + apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") + reconcilerServiceAccount = "etcd-druid-sa" + exemptServiceAccounts = []string{"exempt-sa-1"} -func TestHandleCreate(t *testing.T) { + statefulSetGVK = metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"} + leaseGVK = metav1.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"} +) + +func TestHandleCreateAndConnect(t *testing.T) { g := NewWithT(t) testCases := []struct { - name string - expectedAllowed bool - expectedReason string + name string + operation admissionv1.Operation + expectedMsg string }{ { - name: "create any resource", - expectedAllowed: true, - expectedReason: "operation CREATE is allowed", + name: "allow create operation for any resource", + operation: admissionv1.Create, + expectedMsg: "operation CREATE is allowed", + }, + { + name: "allow connect operation for any resource", + operation: admissionv1.Connect, + expectedMsg: "operation CONNECT is allowed", }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cl := fake.NewClientBuilder().Build() - decoder, err := admission.NewDecoder(cl.Scheme()) - if err != nil { - g.Expect(err).ToNot(HaveOccurred()) - } + cl := testutils.CreateDefaultFakeClient() + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { resp := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Create, + Operation: tc.operation, + // Create for all resources are allowed. StatefulSet resource GVK has been taken as an example. + Kind: statefulSetGVK, }, }) - - g.Expect(resp.Allowed).To(Equal(tc.expectedAllowed)) - g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedReason)) + g.Expect(resp.Allowed).To(BeTrue()) + g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedMsg)) }) } } func TestHandleLeaseUpdate(t *testing.T) { g := NewWithT(t) + cl := fake.NewClientBuilder().Build() + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) - testCases := []struct { - name string - gvk metav1.GroupVersionKind - expectedAllowed bool - expectedReason string - }{ - { - name: "update Lease", - expectedAllowed: true, - expectedReason: "lease resource can be freely updated", + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, }, + decoder: decoder, + logger: logr.Discard(), } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - cl := fake.NewClientBuilder().Build() - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + resp := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Kind: leaseGVK, + }, + }) - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + g.Expect(resp.Allowed).To(BeTrue()) + g.Expect(string(resp.Result.Reason)).To(Equal("lease resource can be freely updated")) +} - resp := handler.Handle(context.Background(), admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Update, - Kind: metav1.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, - }, - }) +func TestUnexpectedResourceType(t *testing.T) { + g := NewWithT(t) - g.Expect(resp.Allowed).To(Equal(tc.expectedAllowed)) - g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedReason)) - }) + cl := fake.NewClientBuilder().Build() + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), } + + resp := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Kind: metav1.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Unknown"}, + }, + }) + + g.Expect(resp.Allowed).To(BeTrue()) + g.Expect(string(resp.Result.Reason)).To(Equal(fmt.Sprintf("unexpected resource type: coordination.k8s.io/Unknown"))) } -func TestHandleUpdate(t *testing.T) { +func TestMissingResourcePartOfLabel(t *testing.T) { g := NewWithT(t) - var ( - reconcilerServiceAccount = "etcd-druid-sa" - exemptServiceAccounts = []string{"exempt-sa-1"} - testUserName = "test-user" - testObjectName = "test" - testNamespace = "test-ns" - testEtcdName = "test" - - internalErr = errors.New("test internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) - apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") - ) + cl := fake.NewClientBuilder().Build() + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), + } + + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{}) + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + UserInfo: authenticationv1.UserInfo{Username: testUserName}, + Kind: statefulSetGVK, + Name: testObjectName, + Namespace: testNamespace, + Object: obj, + OldObject: obj, + }, + }) + + g.Expect(response.Allowed).To(Equal(true)) + g.Expect(string(response.Result.Reason)).To(Equal(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey))) +} + +func TestHandleUpdate(t *testing.T) { + g := NewWithT(t) testCases := []struct { name string @@ -170,51 +206,6 @@ func TestHandleUpdate(t *testing.T) { expectedMessage string expectedCode int32 }{ - { - name: "empty request object", - objectRaw: []byte{}, - expectedAllowed: false, - expectedMessage: "there is no content to decode", - expectedCode: http.StatusInternalServerError, - }, - { - name: "malformed request object", - objectRaw: []byte("foo"), - expectedAllowed: false, - expectedMessage: "invalid character", - expectedCode: http.StatusInternalServerError, - }, - { - name: "resource has no part-of label", - objectLabels: map[string]string{}, - expectedAllowed: true, - expectedReason: fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), - expectedCode: http.StatusOK, - }, - { - name: "etcd not found", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdGetErr: apiNotFoundErr, - expectedAllowed: true, - expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), - expectedCode: http.StatusOK, - }, - { - name: "error in getting etcd", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdGetErr: apiInternalErr, - expectedAllowed: false, - expectedMessage: internalErr.Error(), - expectedCode: http.StatusInternalServerError, - }, - { - name: "etcd reconciliation suspended", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, - expectedAllowed: true, - expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), - expectedCode: http.StatusOK, - }, { name: "resource protection annotation set to false", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, @@ -224,7 +215,7 @@ func TestHandleUpdate(t *testing.T) { expectedCode: http.StatusOK, }, { - name: "etcd is currently being reconciled by druid", + name: "operator makes a request when etcd is being reconciled by druid", userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, @@ -234,7 +225,7 @@ func TestHandleUpdate(t *testing.T) { expectedCode: http.StatusForbidden, }, { - name: "etcd is currently being reconciled by druid, but request is from druid", + name: "druid makes a request during its reconciliation run", userName: reconcilerServiceAccount, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, @@ -265,6 +256,7 @@ func TestHandleUpdate(t *testing.T) { }, } + t.Parallel() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace). @@ -287,21 +279,95 @@ func TestHandleUpdate(t *testing.T) { logger: logr.Discard(), } - sts := &appsv1.StatefulSet{} - obj := runtime.RawExtension{ - Object: buildObject(getObjectGVK(g, sts), testObjectName, testNamespace, tc.objectLabels), - Raw: tc.objectRaw, - } - if tc.objectRaw == nil { - objRaw, err := json.Marshal(obj.Object) - g.Expect(err).ToNot(HaveOccurred()) - obj.Raw = objRaw + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, tc.objectRaw, testObjectName, testNamespace, tc.objectLabels) + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + UserInfo: authenticationv1.UserInfo{Username: tc.userName}, + Kind: statefulSetGVK, + Name: testObjectName, + Namespace: testNamespace, + Object: obj, + OldObject: obj, + }, + }) + + g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(string(response.Result.Reason)).To(Equal(tc.expectedReason)) + g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) + g.Expect(response.Result.Code).To(Equal(tc.expectedCode)) + }) + } +} + +func TestHandleWithInvalidRequestObject(t *testing.T) { + g := NewWithT(t) + testCases := []struct { + name string + operation admissionv1.Operation + objectRaw []byte + expectedAllowed bool + expectedMessage string + expectedErrorCode int32 + }{ + { + name: "empty request object", + operation: admissionv1.Update, + objectRaw: []byte{}, + expectedAllowed: false, + expectedMessage: "there is no content to decode", + expectedErrorCode: http.StatusInternalServerError, + }, + { + name: "malformed request object", + operation: admissionv1.Update, + objectRaw: []byte("foo"), + expectedAllowed: false, + expectedMessage: "invalid character", + expectedErrorCode: http.StatusInternalServerError, + }, + { + name: "empty request object", + operation: admissionv1.Delete, + objectRaw: []byte{}, + expectedAllowed: false, + expectedMessage: "there is no content to decode", + expectedErrorCode: http.StatusInternalServerError, + }, + { + name: "malformed request object", + operation: admissionv1.Delete, + objectRaw: []byte("foo"), + expectedAllowed: false, + expectedMessage: "invalid character", + expectedErrorCode: http.StatusInternalServerError, + }, + } + + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cl := testutils.CreateDefaultFakeClient() + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }, + decoder: decoder, + logger: logr.Discard(), } + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, tc.objectRaw, testObjectName, testNamespace, nil) + response := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, - UserInfo: authenticationv1.UserInfo{Username: tc.userName}, + UserInfo: authenticationv1.UserInfo{Username: testUserName}, Kind: metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, Name: testObjectName, Namespace: testNamespace, @@ -309,6 +375,70 @@ func TestHandleUpdate(t *testing.T) { OldObject: obj, }, }) + g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) + g.Expect(response.Result.Code).To(Equal(tc.expectedErrorCode)) + }) + } +} + +func TestEtcdGetFailures(t *testing.T) { + g := NewWithT(t) + testCases := []struct { + name string + etcdGetErr *apierrors.StatusError + expectedAllowed bool + expectedReason string + expectedMessage string + expectedCode int32 + }{ + { + name: "should allow when etcd is not found", + etcdGetErr: apiNotFoundErr, + expectedAllowed: true, + expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), + expectedCode: http.StatusOK, + }, + { + name: "error in getting etcd", + etcdGetErr: apiInternalErr, + expectedAllowed: false, + expectedMessage: internalErr.Error(), + expectedCode: http.StatusInternalServerError, + }, + } + + t.Parallel() + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() + for _, tc := range testCases { + t.Run(t.Name(), func(t *testing.T) { + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) + decoder, err := admission.NewDecoder(cl.Scheme()) + g.Expect(err).ToNot(HaveOccurred()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }, + decoder: decoder, + logger: logr.Discard(), + } + + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}) + + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + UserInfo: authenticationv1.UserInfo{Username: testUserName}, + Kind: metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}, + Name: testObjectName, + Namespace: testNamespace, + OldObject: obj, + }, + }) g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) g.Expect(string(response.Result.Reason)).To(Equal(tc.expectedReason)) @@ -321,19 +451,6 @@ func TestHandleUpdate(t *testing.T) { func TestHandleDelete(t *testing.T) { g := NewWithT(t) - var ( - reconcilerServiceAccount = "etcd-druid-sa" - exemptServiceAccounts = []string{"exempt-sa-1"} - testUserName = "test-user" - testObjectName = "test" - testNamespace = "test-ns" - testEtcdName = "test" - - internalErr = errors.New("test internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) - apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") - ) - testCases := []struct { name string // ----- request ----- @@ -353,43 +470,6 @@ func TestHandleDelete(t *testing.T) { expectedMessage string expectedCode int32 }{ - { - name: "empty request object", - objectRaw: []byte{}, - expectedAllowed: false, - expectedMessage: "there is no content to decode", - expectedCode: http.StatusInternalServerError, - }, - { - name: "malformed request object", - objectRaw: []byte("foo"), - expectedAllowed: false, - expectedMessage: "invalid character", - expectedCode: http.StatusInternalServerError, - }, - { - name: "resource has no part-of label", - objectLabels: map[string]string{}, - expectedAllowed: true, - expectedReason: fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), - expectedCode: http.StatusOK, - }, - { - name: "etcd not found", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdGetErr: apiNotFoundErr, - expectedAllowed: true, - expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), - expectedCode: http.StatusOK, - }, - { - name: "error in getting etcd", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdGetErr: apiInternalErr, - expectedAllowed: false, - expectedMessage: internalErr.Error(), - expectedCode: http.StatusInternalServerError, - }, { name: "etcd reconciliation suspended", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, @@ -470,16 +550,7 @@ func TestHandleDelete(t *testing.T) { logger: logr.Discard(), } - sts := &appsv1.StatefulSet{} - obj := runtime.RawExtension{ - Object: buildObject(getObjectGVK(g, sts), testObjectName, testNamespace, tc.objectLabels), - Raw: tc.objectRaw, - } - if tc.objectRaw == nil { - objRaw, err := json.Marshal(obj.Object) - g.Expect(err).ToNot(HaveOccurred()) - obj.Raw = objRaw - } + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, tc.objectRaw, testObjectName, testNamespace, tc.objectLabels) response := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ @@ -499,3 +570,37 @@ func TestHandleDelete(t *testing.T) { }) } } + +// ---------------- Helper functions ------------------- + +func buildObjRawExtension(g *WithT, emptyObj runtime.Object, objRaw []byte, testObjectName, testNs string, labels map[string]string) runtime.RawExtension { + var ( + rawBytes []byte + err error + ) + rawBytes = objRaw + obj := buildObject(getObjectGVK(g, emptyObj), testObjectName, testNs, labels) + if objRaw == nil { + rawBytes, err = json.Marshal(obj) + g.Expect(err).ToNot(HaveOccurred()) + } + return runtime.RawExtension{ + Object: obj, + Raw: rawBytes, + } +} + +func getObjectGVK(g *WithT, obj runtime.Object) schema.GroupVersionKind { + gvk, err := apiutil.GVKForObject(obj, kubernetes.Scheme) + g.Expect(err).ToNot(HaveOccurred()) + return gvk +} + +func buildObject(gvk schema.GroupVersionKind, name, namespace string, labels map[string]string) runtime.Object { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetName(name) + obj.SetNamespace(namespace) + obj.SetLabels(labels) + return obj +} diff --git a/test/utils/client.go b/test/utils/client.go index 996b3ad49..cabef09d1 100644 --- a/test/utils/client.go +++ b/test/utils/client.go @@ -59,6 +59,11 @@ type TestClientBuilder struct { errorRecords []errorRecord } +// CreateDefaultFakeClient returns a default fake client.Client without any configured reactions to errors. +func CreateDefaultFakeClient() client.Client { + return fake.NewClientBuilder().Build() +} + // CreateTestFakeClientForObjects is a convenience function which creates a test client which uses a fake client as a delegate and reacts to the configured errors for the given object key. func CreateTestFakeClientForObjects(getErr, createErr, patchErr, deleteErr *apierrors.StatusError, existingObjects []client.Object, objKeys ...client.ObjectKey) client.Client { fakeDelegateClientBuilder := fake.NewClientBuilder() From b4c0b056b1656b86851b514edd17e810e990b7d2 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sun, 31 Mar 2024 17:51:35 +0530 Subject: [PATCH 123/235] removed an unnecessary unit test --- internal/webhook/sentinel/handler_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index b51d31355..88b3e18de 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -470,14 +470,6 @@ func TestHandleDelete(t *testing.T) { expectedMessage string expectedCode int32 }{ - { - name: "etcd reconciliation suspended", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdAnnotations: map[string]string{druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true"}, - expectedAllowed: true, - expectedReason: fmt.Sprintf("spec reconciliation of etcd %s is currently suspended", testEtcdName), - expectedCode: http.StatusOK, - }, { name: "resource protection annotation set to false", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, From da7a5147731cfa6e310e8a3f3828b75cbdb87857 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 2 Apr 2024 11:16:03 +0530 Subject: [PATCH 124/235] Fix volume mount paths, and other minor fixes --- internal/controller/compaction/reconciler.go | 2 +- internal/controller/secret/reconciler.go | 7 + internal/operator/configmap/configmap_test.go | 10 +- internal/operator/configmap/etcdconfig.go | 18 +- internal/operator/statefulset/builder.go | 169 ++++++++++-------- internal/operator/statefulset/constants.go | 42 ++--- internal/operator/statefulset/statefulset.go | 8 +- internal/operator/statefulset/stsmatcher.go | 42 ++--- internal/utils/lease.go | 12 +- internal/utils/lease_test.go | 2 +- test/e2e/etcd_backup_test.go | 8 +- test/e2e/utils.go | 70 ++++---- 12 files changed, 218 insertions(+), 172 deletions(-) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 6b3553587..ed4f20a8a 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -104,7 +104,7 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd job := &batchv1.Job{} if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { if errors.IsNotFound(err) { - logger.Info("Currently, no compaction job is running in the namespace ", etcd.Namespace) + logger.Info("No compaction job currently running", "namespace", etcd.Namespace) } else { // Error reading the object - requeue the request. return ctrl.Result{ diff --git a/internal/controller/secret/reconciler.go b/internal/controller/secret/reconciler.go index 81eda1902..cc892b2a8 100644 --- a/internal/controller/secret/reconciler.go +++ b/internal/controller/secret/reconciler.go @@ -57,6 +57,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } if needed, etcd := isFinalizerNeeded(secret.Name, etcdList); needed { + if hasFinalizer(secret) { + return ctrl.Result{}, nil + } logger.Info("Adding finalizer for secret since it is referenced by etcd resource", "secretNamespace", secret.Namespace, "secretName", secret.Name, "etcdNamespace", etcd.Namespace, "etcdName", etcd.Name) return ctrl.Result{}, addFinalizer(ctx, logger, r.Client, secret) @@ -89,6 +92,10 @@ func isFinalizerNeeded(secretName string, etcdList *druidv1alpha1.EtcdList) (boo return false, nil } +func hasFinalizer(secret *corev1.Secret) bool { + return sets.NewString(secret.Finalizers...).Has(common.FinalizerName) +} + func addFinalizer(ctx context.Context, logger logr.Logger, k8sClient client.Client, secret *corev1.Secret) error { if finalizers := sets.NewString(secret.Finalizers...); finalizers.Has(common.FinalizerName) { return nil diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 797f90e52..8a584e2da 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -14,8 +14,10 @@ import ( "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/component" + "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" @@ -334,7 +336,7 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C g.Expect(err).ToNot(HaveOccurred()) g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "name": Equal(fmt.Sprintf("etcd-%s", etcd.UID[:6])), - "data-dir": Equal("/var/etcd/data/new.etcd"), + "data-dir": Equal(fmt.Sprintf("%s/new.etcd", statefulset.EtcdDataVolumeMountPath)), "metrics": Equal(string(druidv1alpha1.Basic)), "snapshot-count": Equal(int64(75000)), "enable-v2": Equal(false), @@ -354,10 +356,10 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ - "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), - "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), + "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/server/tls.key"), "client-cert-auth": Equal(true), - "trusted-ca-file": Equal("/var/etcd/ssl/client/ca/ca.crt"), + "trusted-ca-file": Equal("/var/etcd/ssl/ca/ca.crt"), "auto-tls": Equal(false), }), })) diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index cea8cf404..89401c608 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -10,6 +10,7 @@ import ( "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/etcd-druid/internal/utils" "k8s.io/utils/pointer" ) @@ -18,7 +19,6 @@ import ( const ( defaultDBQuotaBytes = int64(8 * 1024 * 1024 * 1024) // 8Gi defaultAutoCompactionRetention = "30m" - defaultDataDir = "/var/etcd/data/new.etcd" defaultInitialClusterToken = "etcd-cluster" defaultInitialClusterState = "new" // For more information refer to https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention @@ -29,6 +29,10 @@ const ( defaultServerPort = 2380 ) +var ( + defaultDataDir = fmt.Sprintf("%s/new.etcd", statefulset.EtcdDataVolumeMountPath) +) + type tlsTarget string const ( @@ -65,8 +69,8 @@ type securityConfig struct { } func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { - peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, peerTLS) - clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, clientTLS) + clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, statefulset.EtcdCAVolumeMountPath, statefulset.EtcdServerTLSVolumeMountPath) + peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, statefulset.EtcdPeerCAVolumeMountPath, statefulset.EtcdPeerServerTLSVolumeMountPath) cfg := &etcdConfig{ Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), @@ -103,14 +107,14 @@ func getDBQuotaBytes(etcd *druidv1alpha1.Etcd) int64 { return dbQuotaBytes } -func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, tlsTarget tlsTarget) (string, *securityConfig) { +func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, caPath, serverTLSPath string) (string, *securityConfig) { if tlsConfig != nil { const defaultTLSCASecretKey = "ca.crt" return "https", &securityConfig{ - CertFile: fmt.Sprintf("/var/etcd/ssl/%s/server/tls.crt", tlsTarget), - KeyFile: fmt.Sprintf("/var/etcd/ssl/%s/server/tls.key", tlsTarget), + CertFile: fmt.Sprintf("%s/tls.crt", serverTLSPath), + KeyFile: fmt.Sprintf("%s/tls.key", serverTLSPath), ClientCertAuth: true, - TrustedCAFile: fmt.Sprintf("/var/etcd/ssl/%s/ca/%s", tlsTarget, utils.TypeDeref[string](tlsConfig.TLSCASecretRef.DataKey, defaultTLSCASecretKey)), + TrustedCAFile: fmt.Sprintf("%s/%s", caPath, utils.TypeDeref[string](tlsConfig.TLSCASecretRef.DataKey, defaultTLSCASecretKey)), AutoTLS: false, } } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index d5a967795..afc9434d2 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -222,7 +222,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, - Args: []string{"chown -R 65532:65532 /var/etcd/data"}, + Args: []string{fmt.Sprintf("chown -R 65532:65532 %s", EtcdDataVolumeMountPath)}, VolumeMounts: []corev1.VolumeMount{b.getEtcdDataVolumeMount()}, SecurityContext: &corev1.SecurityContext{ RunAsGroup: pointer.Int64(0), @@ -232,28 +232,31 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { }) if b.etcd.IsBackupStoreEnabled() { if b.provider != nil && *b.provider == utils.Local { - initContainers = append(initContainers, corev1.Container{ - Name: common.ChangeBackupBucketPermissionsInitContainerName, - Image: b.initContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *b.etcd.Spec.Backup.Store.Container)}, - VolumeMounts: b.getBackupRestoreContainerVolumeMounts(), - SecurityContext: &corev1.SecurityContext{ - RunAsGroup: pointer.Int64(0), - RunAsNonRoot: pointer.Bool(false), - RunAsUser: pointer.Int64(0), - }, - }) + etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() + if etcdBackupVolumeMount != nil { + initContainers = append(initContainers, corev1.Container{ + Name: common.ChangeBackupBucketPermissionsInitContainerName, + Image: b.initContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", "--"}, + Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *b.etcd.Spec.Backup.Store.Container)}, + VolumeMounts: []corev1.VolumeMount{*etcdBackupVolumeMount}, + SecurityContext: &corev1.SecurityContext{ + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + RunAsUser: pointer.Int64(0), + }, + }) + } } } return initContainers } func (b *stsBuilder) getEtcdContainerVolumeMounts() []corev1.VolumeMount { - etcdVolumeMounts := make([]corev1.VolumeMount, 0, 6) + etcdVolumeMounts := make([]corev1.VolumeMount, 0, 7) etcdVolumeMounts = append(etcdVolumeMounts, b.getEtcdDataVolumeMount()) - etcdVolumeMounts = append(etcdVolumeMounts, b.getEtcdSecretVolumeMounts()...) + etcdVolumeMounts = append(etcdVolumeMounts, b.getEtcdContainerSecretVolumeMounts()...) return etcdVolumeMounts } @@ -266,7 +269,7 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun MountPath: etcdConfigFileMountPath, }, ) - brVolumeMounts = append(brVolumeMounts, b.getBackupRestoreSecretVolumeMounts()...) + brVolumeMounts = append(brVolumeMounts, b.getBackupRestoreContainerSecretVolumeMounts()...) if b.etcd.IsBackupStoreEnabled() { etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() @@ -277,24 +280,30 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun return brVolumeMounts } -func (b *stsBuilder) getBackupRestoreSecretVolumeMounts() []corev1.VolumeMount { +func (b *stsBuilder) getBackupRestoreContainerSecretVolumeMounts() []corev1.VolumeMount { + secretVolumeMounts := make([]corev1.VolumeMount, 0, 3) if b.etcd.Spec.Backup.TLS != nil { - return []corev1.VolumeMount{ - { - Name: backRestoreCAVolumeName, - MountPath: backupRestoreCAVolumeMountPath, - }, - { - Name: backRestoreServerTLSVolumeName, + secretVolumeMounts = append(secretVolumeMounts, + corev1.VolumeMount{ + Name: backupRestoreServerTLSVolumeName, MountPath: backupRestoreServerTLSVolumeMountPath, }, - { - Name: backRestoreClientTLSVolumeName, - MountPath: backupRestoreClientTLSVolumeMountPath, + ) + } + if b.etcd.Spec.Etcd.ClientUrlTLS != nil { + secretVolumeMounts = append(secretVolumeMounts, + corev1.VolumeMount{ + Name: etcdCAVolumeName, + MountPath: EtcdCAVolumeMountPath, }, - } + corev1.VolumeMount{ + Name: etcdClientTLSVolumeName, + MountPath: etcdClientTLSVolumeMountPath, + }, + ) } - return []corev1.VolumeMount{} + + return secretVolumeMounts } func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { @@ -331,7 +340,7 @@ func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) return corev1.VolumeMount{ Name: volumeClaimTemplateName, - MountPath: etcdDataVolumeMountPath, + MountPath: EtcdDataVolumeMountPath, } } @@ -427,10 +436,10 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { // Client and Backup TLS command line args // ----------------------------------------------------------------------------------------------------------------- if b.etcd.Spec.Etcd.ClientUrlTLS != nil { - commandArgs = append(commandArgs, "--cert=/var/etcd/ssl/client/client/tls.crt") - commandArgs = append(commandArgs, "--key=/var/etcd/ssl/client/client/tls.key") dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - commandArgs = append(commandArgs, fmt.Sprintf("--cacert=/var/etcd/ssl/client/ca/%s", dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--cacert=%s/%s", EtcdCAVolumeMountPath, dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--cert=%s/tls.crt", etcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--key=%s/tls.key", etcdClientTLSVolumeMountPath)) commandArgs = append(commandArgs, "--insecure-transport=false") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=false") commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) @@ -448,9 +457,9 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { // Other misc command line args // ----------------------------------------------------------------------------------------------------------------- - commandArgs = append(commandArgs, "--data-dir=/var/etcd/data/new.etcd") - commandArgs = append(commandArgs, "--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp") - commandArgs = append(commandArgs, "--snapstore-temp-directory=/var/etcd/data/temp") + commandArgs = append(commandArgs, fmt.Sprintf("--data-dir=%s/new.etcd", EtcdDataVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--restoration-temp-snapshots-dir=%s/restoration.temp", EtcdDataVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--snapstore-temp-directory=%s/temp", EtcdDataVolumeMountPath)) commandArgs = append(commandArgs, fmt.Sprintf("--etcd-connection-timeout=%s", defaultEtcdConnectionTimeout)) commandArgs = append(commandArgs, "--enable-member-lease-renewal=true") @@ -580,9 +589,9 @@ func (b *stsBuilder) getEtcdContainerReadinessProbeCommand() []string { cmdBuilder.WriteString("ETCDCTL_API=3 etcdctl") if b.etcd.Spec.Etcd.ClientUrlTLS != nil { dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - cmdBuilder.WriteString(" --cacert=/var/etcd/ssl/client/ca/" + dataKey) - cmdBuilder.WriteString(" --cert=/var/etcd/ssl/client/client/tls.crt") - cmdBuilder.WriteString(" --key=/var/etcd/ssl/client/client/tls.key") + cmdBuilder.WriteString(fmt.Sprintf(" --cacert=%s/%s", EtcdCAVolumeMountPath, dataKey)) + cmdBuilder.WriteString(fmt.Sprintf(" --cert=%s/tls.crt", etcdClientTLSVolumeMountPath)) + cmdBuilder.WriteString(fmt.Sprintf(" --key=%s/tls.key", etcdClientTLSVolumeMountPath)) cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) } else { cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", b.etcd.Name, b.clientPort)) @@ -609,11 +618,11 @@ func (b *stsBuilder) getEtcdContainerCommandArgs() []string { if b.etcd.Spec.Etcd.ClientUrlTLS == nil { commandArgs = append(commandArgs, "--backup-restore-tls-enabled=false") } else { - dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") commandArgs = append(commandArgs, "--backup-restore-tls-enabled=true") - commandArgs = append(commandArgs, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") - commandArgs = append(commandArgs, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") - commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) + dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") + commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=%s/%s", backupRestoreCAVolumeMountPath, dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-cert-path=%s/tls.crt", etcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-key-path=%s/tls.key", etcdClientTLSVolumeMountPath)) } return commandArgs } @@ -640,32 +649,48 @@ func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { RunAsGroup: pointer.Int64(65532), RunAsNonRoot: pointer.Bool(true), RunAsUser: pointer.Int64(65532), - FSGroup: pointer.Int64(65532), + // TODO: is this necessary? + FSGroup: pointer.Int64(65532), } } -func (b *stsBuilder) getEtcdSecretVolumeMounts() []corev1.VolumeMount { - secretVolumeMounts := make([]corev1.VolumeMount, 0, 5) +func (b *stsBuilder) getEtcdContainerSecretVolumeMounts() []corev1.VolumeMount { + secretVolumeMounts := make([]corev1.VolumeMount, 0, 6) if b.etcd.Spec.Etcd.ClientUrlTLS != nil { - secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: clientCAVolumeName, - MountPath: etcdCAVolumeMountPath, - }, corev1.VolumeMount{ - Name: serverTLSVolumeName, - MountPath: etcdServerTLSVolumeMountPath, - }, corev1.VolumeMount{ - Name: clientTLSVolumeName, - MountPath: etcdClientTLSVolumeMountPath, - }) + secretVolumeMounts = append(secretVolumeMounts, + corev1.VolumeMount{ + Name: etcdCAVolumeName, + MountPath: EtcdCAVolumeMountPath, + }, + corev1.VolumeMount{ + Name: etcdServerTLSVolumeName, + MountPath: EtcdServerTLSVolumeMountPath, + }, + corev1.VolumeMount{ + Name: etcdClientTLSVolumeName, + MountPath: etcdClientTLSVolumeMountPath, + }, + ) } if b.etcd.Spec.Etcd.PeerUrlTLS != nil { - secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: peerCAVolumeName, - MountPath: etcdPeerCAVolumeMountPath, - }, corev1.VolumeMount{ - Name: peerServerTLSVolumeName, - MountPath: etcdPeerServerTLSVolumeMountPath, - }) + secretVolumeMounts = append(secretVolumeMounts, + corev1.VolumeMount{ + Name: etcdPeerCAVolumeName, + MountPath: EtcdPeerCAVolumeMountPath, + }, + corev1.VolumeMount{ + Name: etcdPeerServerTLSVolumeName, + MountPath: EtcdPeerServerTLSVolumeMountPath, + }, + ) + } + if b.etcd.Spec.Backup.TLS != nil { + secretVolumeMounts = append(secretVolumeMounts, + corev1.VolumeMount{ + Name: backupRestoreCAVolumeName, + MountPath: backupRestoreCAVolumeMountPath, + }, + ) } return secretVolumeMounts } @@ -717,7 +742,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { clientTLSConfig := b.etcd.Spec.Etcd.ClientUrlTLS return []corev1.Volume{ { - Name: clientCAVolumeName, + Name: etcdCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.TLSCASecretRef.Name, @@ -726,7 +751,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { }, }, { - Name: serverTLSVolumeName, + Name: etcdServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ServerTLSSecretRef.Name, @@ -735,7 +760,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { }, }, { - Name: clientTLSVolumeName, + Name: etcdClientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ClientTLSSecretRef.Name, @@ -750,7 +775,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { peerTLSConfig := b.etcd.Spec.Etcd.PeerUrlTLS return []corev1.Volume{ { - Name: peerCAVolumeName, + Name: etcdPeerCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.TLSCASecretRef.Name, @@ -759,7 +784,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { }, }, { - Name: peerServerTLSVolumeName, + Name: etcdPeerServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.ServerTLSSecretRef.Name, @@ -774,7 +799,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { tlsConfig := b.etcd.Spec.Backup.TLS return []corev1.Volume{ { - Name: backRestoreCAVolumeName, + Name: backupRestoreCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.TLSCASecretRef.Name, @@ -783,7 +808,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { }, }, { - Name: backRestoreServerTLSVolumeName, + Name: backupRestoreServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ServerTLSSecretRef.Name, @@ -792,7 +817,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { }, }, { - Name: backRestoreClientTLSVolumeName, + Name: backupRestoreClientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ClientTLSSecretRef.Name, diff --git a/internal/operator/statefulset/constants.go b/internal/operator/statefulset/constants.go index 7cc2d4a99..90dc26b91 100644 --- a/internal/operator/statefulset/constants.go +++ b/internal/operator/statefulset/constants.go @@ -6,32 +6,32 @@ package statefulset // constants for volume names const ( - // clientCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. - clientCAVolumeName = "ca" - // serverTLSVolumeName is the name of the volume that contains the server certificate used to set up the server (etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). - serverTLSVolumeName = "server-tls" - // clientTLSVolumeName is the name of the volume that contains the client certificate used by the client to communicate to the server(etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). - clientTLSVolumeName = "client-tls" - peerCAVolumeName = "peer-ca" - peerServerTLSVolumeName = "peer-server-tls" - backRestoreCAVolumeName = "back-restore-ca" - backRestoreServerTLSVolumeName = "back-restore-server-tls" - backRestoreClientTLSVolumeName = "back-restore-client-tls" - etcdConfigVolumeName = "etcd-config-file" - localBackupVolumeName = "local-backup" - providerBackupVolumeName = "etcd-backup" + // etcdCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. + etcdCAVolumeName = "etcd-ca" + // etcdServerTLSVolumeName is the name of the volume that contains the server certificate used to set up the server (etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). + etcdServerTLSVolumeName = "etcd-server-tls" + // etcdClientTLSVolumeName is the name of the volume that contains the client certificate used by the client to communicate to the server(etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). + etcdClientTLSVolumeName = "etcd-client-tls" + etcdPeerCAVolumeName = "etcd-peer-ca" + etcdPeerServerTLSVolumeName = "etcd-peer-server-tls" + backupRestoreCAVolumeName = "backup-restore-ca" + backupRestoreServerTLSVolumeName = "backup-restore-server-tls" + backupRestoreClientTLSVolumeName = "backup-restore-client-tls" + etcdConfigVolumeName = "etcd-config-file" + localBackupVolumeName = "local-backup" + providerBackupVolumeName = "etcd-backup" ) // constants for volume mount paths const ( + EtcdCAVolumeMountPath = "/var/etcd/ssl/ca" + EtcdServerTLSVolumeMountPath = "/var/etcd/ssl/server" + etcdClientTLSVolumeMountPath = "/var/etcd/ssl/client" + EtcdPeerCAVolumeMountPath = "/var/etcd/ssl/peer/ca" + EtcdPeerServerTLSVolumeMountPath = "/var/etcd/ssl/peer/server" backupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" backupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" backupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" - etcdCAVolumeMountPath = "/var/etcd/ssl/ca" - etcdServerTLSVolumeMountPath = "/var/etcd/ssl/server" - etcdClientTLSVolumeMountPath = "/var/etcd/ssl/client" - etcdPeerCAVolumeMountPath = "/var/etcd/ssl/peer/ca" - etcdPeerServerTLSVolumeMountPath = "/var/etcd/ssl/peer/server" gcsBackupVolumeMountPath = "/var/.gcp/" nonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" ) @@ -42,8 +42,8 @@ const ( ) const ( - // etcdDataVolumeMountPath is the path on etcd and etcd-backup-restore containers where etcd data directory is hosted. - etcdDataVolumeMountPath = "/var/etcd/data" + // EtcdDataVolumeMountPath is the path on etcd and etcd-backup-restore containers where etcd data directory is hosted. + EtcdDataVolumeMountPath = "/var/etcd/data" ) // constants for container ports diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index af00bfe15..e2e93bb1a 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -158,7 +158,7 @@ func (r _resource) createOrPatch(ctx component.OperatorContext, etcd *druidv1alp } func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - peerTLSEnabledForAllMembers, err := utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) + peerTLSEnabledForAllMembers, err := utils.IsPeerURLTLSEnabledForMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name, int(*existingSts.Spec.Replicas)) if err != nil { return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) } @@ -174,7 +174,7 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru ctx.Logger.Info("Secret volume mounts to enable Peer URL TLS have already been mounted. Skipping patching StatefulSet with secret volume mounts.") } // check again if peer TLS has been enabled for all members. If not then force a requeue of the reconcile request. - peerTLSEnabledForAllMembers, err = utils.IsPeerURLTLSEnabledForAllMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name) + peerTLSEnabledForAllMembers, err = utils.IsPeerURLTLSEnabledForMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name, int(*existingSts.Spec.Replicas)) if err != nil { return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) } @@ -190,10 +190,10 @@ func isStatefulSetPatchedWithPeerTLSVolMount(existingSts *appsv1.StatefulSet) bo volumes := existingSts.Spec.Template.Spec.Volumes var peerURLCAEtcdVolPresent, peerURLEtcdServerTLSVolPresent bool for _, vol := range volumes { - if vol.Name == "peer-url-ca-etcd" { + if vol.Name == etcdPeerCAVolumeName { peerURLCAEtcdVolPresent = true } - if vol.Name == "peer-url-etcd-server-tls" { + if vol.Name == etcdPeerServerTLSVolumeName { peerURLEtcdServerTLSVolPresent = true } } diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index d6246bf70..c424261cc 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -292,7 +292,7 @@ func (s StatefulSetMatcher) matchEtcdContainerReadinessProbeCmd() gomegatypes.Go return HaveExactElements( "/bin/sh", "-ec", - fmt.Sprintf("ETCDCTL_API=3 etcdctl --cacert=/var/etcd/ssl/client/ca/%s --cert=/var/etcd/ssl/client/client/tls.crt --key=/var/etcd/ssl/client/client/tls.key --endpoints=https://%s-local:%d get foo --consistency=l", dataKey, s.etcd.Name, s.clientPort), + fmt.Sprintf("ETCDCTL_API=3 etcdctl --cacert=/var/etcd/ssl/ca/%s --cert=/var/etcd/ssl/client/tls.crt --key=/var/etcd/ssl/client/tls.key --endpoints=https://%s-local:%d get foo --consistency=l", dataKey, s.etcd.Name, s.clientPort), ) } else { return HaveExactElements( @@ -316,16 +316,16 @@ func (s StatefulSetMatcher) matchEtcdContainerCmdArgs() gomegatypes.GomegaMatche } else { dataKey := utils.TypeDeref(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") cmdArgs = append(cmdArgs, "--backup-restore-tls-enabled=true") - cmdArgs = append(cmdArgs, "--etcd-client-cert-path=/var/etcd/ssl/client/client/tls.crt") - cmdArgs = append(cmdArgs, "--etcd-client-key-path=/var/etcd/ssl/client/client/tls.key") - cmdArgs = append(cmdArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcd/ssl/client/ca/%s", dataKey)) + cmdArgs = append(cmdArgs, "--etcd-client-cert-path=/var/etcd/ssl/client/tls.crt") + cmdArgs = append(cmdArgs, "--etcd-client-key-path=/var/etcd/ssl/client/tls.key") + cmdArgs = append(cmdArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=/var/etcdbr/ssl/ca/%s", dataKey)) } return HaveExactElements(cmdArgs) } func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) - return matchVolMount(volumeClaimTemplateName, etcdDataVolumeMountPath) + return matchVolMount(volumeClaimTemplateName, EtcdDataVolumeMountPath) } func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.GomegaMatcher { @@ -345,9 +345,9 @@ func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.G func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 3) if s.etcd.Spec.Backup.TLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backRestoreCAVolumeName, backupRestoreCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backRestoreServerTLSVolumeName, backupRestoreServerTLSVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backRestoreClientTLSVolumeName, backupRestoreClientTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backupRestoreCAVolumeName, backupRestoreCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backupRestoreServerTLSVolumeName, backupRestoreServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backupRestoreClientTLSVolumeName, backupRestoreClientTLSVolumeMountPath)) } return secretVolMountMatchers } @@ -355,13 +355,13 @@ func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatyp func (s StatefulSetMatcher) getEtcdSecretVolMountsMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(clientCAVolumeName, etcdCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(serverTLSVolumeName, etcdServerTLSVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(clientTLSVolumeName, etcdClientTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdCAVolumeName, EtcdCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdServerTLSVolumeName, EtcdServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdClientTLSVolumeName, etcdClientTLSVolumeMountPath)) } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(peerCAVolumeName, etcdPeerCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(peerServerTLSVolumeName, etcdPeerServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdPeerCAVolumeName, EtcdPeerCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdPeerServerTLSVolumeName, EtcdPeerServerTLSVolumeMountPath)) } return secretVolMountMatchers } @@ -448,7 +448,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM volMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(clientCAVolumeName), + "Name": Equal(etcdCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), @@ -457,7 +457,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(serverTLSVolumeName), + "Name": Equal(etcdServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), @@ -466,7 +466,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(clientTLSVolumeName), + "Name": Equal(etcdClientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), @@ -477,7 +477,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(peerCAVolumeName), + "Name": Equal(etcdPeerCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), @@ -487,7 +487,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(peerServerTLSVolumeName), + "Name": Equal(etcdPeerServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), @@ -498,7 +498,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM } if s.etcd.Spec.Backup.TLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(backRestoreCAVolumeName), + "Name": Equal(backupRestoreCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), @@ -508,7 +508,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(backRestoreServerTLSVolumeName), + "Name": Equal(backupRestoreServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), @@ -518,7 +518,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(backRestoreClientTLSVolumeName), + "Name": Equal(backupRestoreClientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), diff --git a/internal/utils/lease.go b/internal/utils/lease.go index d56f442ab..b352d77bf 100644 --- a/internal/utils/lease.go +++ b/internal/utils/lease.go @@ -6,7 +6,9 @@ package utils import ( "context" + "slices" "strconv" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" @@ -17,8 +19,8 @@ import ( const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" -// IsPeerURLTLSEnabledForAllMembers checks if TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. -func IsPeerURLTLSEnabledForAllMembers(ctx context.Context, cl client.Client, logger logr.Logger, namespace, etcdName string) (bool, error) { +// IsPeerURLTLSEnabledForMembers checks if TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. +func IsPeerURLTLSEnabledForMembers(ctx context.Context, cl client.Client, logger logr.Logger, namespace, etcdName string, numReplicas int) (bool, error) { leaseList := &coordinationv1.LeaseList{} if err := cl.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(map[string]string{ druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, @@ -28,7 +30,11 @@ func IsPeerURLTLSEnabledForAllMembers(ctx context.Context, cl client.Client, log return false, err } tlsEnabledForAllMembers := true - for _, lease := range leaseList.Items { + leases := leaseList.DeepCopy().Items + slices.SortFunc(leases, func(a, b coordinationv1.Lease) int { + return strings.Compare(a.Name, b.Name) + }) + for _, lease := range leases[:numReplicas] { tlsEnabled, err := parseAndGetTLSEnabledValue(lease, logger) if err != nil { return false, err diff --git a/internal/utils/lease_test.go b/internal/utils/lease_test.go index 88e29d328..082612637 100644 --- a/internal/utils/lease_test.go +++ b/internal/utils/lease_test.go @@ -71,7 +71,7 @@ func TestIsPeerURLTLSEnabledForAllMembers(t *testing.T) { druidv1alpha1.LabelPartOfKey: testutils.TestEtcdName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, }, existingObjects...) - tlsEnabled, err := IsPeerURLTLSEnabledForAllMembers(context.Background(), cl, logger, testutils.TestNamespace, testutils.TestEtcdName) + tlsEnabled, err := IsPeerURLTLSEnabledForMembers(context.Background(), cl, logger, testutils.TestNamespace, testutils.TestEtcdName, etcdReplicas) if tc.expectedErr != nil { g.Expect(err).To(Equal(tc.expectedErr)) } else { diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index 3fb5fa40f..337b4e1aa 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -11,7 +11,6 @@ import ( brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/gardener/pkg/utils/test/matchers" "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" @@ -222,8 +221,11 @@ func checkEtcdReady(ctx context.Context, cl client.Client, logger logr.Logger, e } // Ensure the etcd cluster's current generation matches the observed generation - if etcd.Status.ObservedGeneration != &etcd.Generation { - return fmt.Errorf("etcd '%s' is not at the expected generation (observed: %d, expected: %d)", etcd.Name, etcd.Status.ObservedGeneration, etcd.Generation) + if etcd.Status.ObservedGeneration == nil { + return fmt.Errorf("etcd %s status observed generation is nil", etcd.Name) + } + if *etcd.Status.ObservedGeneration != etcd.Generation { + return fmt.Errorf("etcd '%s' is not at the expected generation (observed: %d, expected: %d)", etcd.Name, *etcd.Status.ObservedGeneration, etcd.Generation) } if etcd.Status.Ready == nil || *etcd.Status.Ready != true { diff --git a/test/e2e/utils.go b/test/e2e/utils.go index ab58a4b31..4a4878926 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -576,7 +576,7 @@ func getRemoteCommandExecutor(kubeconfigPath, namespace, podName, containerName, // executeRemoteCommand executes a remote shell command on the given pod and container // and returns the stdout and stderr logs -func executeRemoteCommand(ctx context.Context, kubeconfigPath, namespace, podName, containerName, command string) (string, string, error) { +func executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, command string) (string, string, error) { exec, err := getRemoteCommandExecutor(kubeconfigPath, namespace, podName, containerName, command) if err != nil { return "", "", err @@ -584,7 +584,7 @@ func executeRemoteCommand(ctx context.Context, kubeconfigPath, namespace, podNam buf := &bytes.Buffer{} errBuf := &bytes.Buffer{} - err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + err = exec.Stream(remotecommand.StreamOptions{ Stdout: buf, Stderr: errBuf, }) @@ -663,7 +663,7 @@ func getPurgeLocalSnapstoreJob(storeContainer string) *batchv1.Job { ) } -func populateEtcd(ctx context.Context, logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, valuePrefix string, startKeyNo, endKeyNo int, delay time.Duration) error { +func populateEtcd(logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, valuePrefix string, startKeyNo, endKeyNo int, delay time.Duration) error { var ( cmd string stdout string @@ -679,14 +679,14 @@ func populateEtcd(ctx context.Context, logger logr.Logger, kubeconfigPath, names "rm -rf etcd-$ETCD_VERSION-linux-amd64;'" + " > test.sh && sh test.sh" - stdout, stderr, err = executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, install) + stdout, stderr, err = executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, install) if err != nil { logger.Error(err, fmt.Sprintf("Failed to inatall etcdctl. err: %s, stderr: %s, stdout:%s", err, stderr, stdout)) } for i := startKeyNo; i <= endKeyNo; { - cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://%s-client.shoot.svc:%d --cacert /var/etcd/ssl/client/ca/ca.crt --cert=/var/etcd/ssl/client/client/tls.crt --key=/var/etcd/ssl/client/client/tls.key put %s-%d %s-%d", etcdName, etcdClientPort, keyPrefix, i, valuePrefix, i) - stdout, stderr, err = executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) + cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://%s-client.shoot.svc:%d --cacert /var/etcd/ssl/ca/ca.crt --cert=/var/etcd/ssl/client/tls.crt --key=/var/etcd/ssl/client/tls.key put %s-%d %s-%d", etcdName, etcdClientPort, keyPrefix, i, valuePrefix, i) + stdout, stderr, err = executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stderr != "" || stdout != "OK" { logger.Error(err, fmt.Sprintf("failed to put (%s-%d, %s-%d): stdout: %s; stderr: %s. Retrying", keyPrefix, i, valuePrefix, i, stdout, stderr)) retries++ @@ -703,7 +703,7 @@ func populateEtcd(ctx context.Context, logger logr.Logger, kubeconfigPath, names return nil } -func getEtcdKey(ctx context.Context, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, suffix int) (string, string, error) { +func getEtcdKey(kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, suffix int) (string, string, error) { var ( cmd string stdout string @@ -711,8 +711,8 @@ func getEtcdKey(ctx context.Context, kubeconfigPath, namespace, etcdName, podNam err error ) - cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://%s-client.shoot.svc:%d --cacert /var/etcd/ssl/client/ca/ca.crt --cert=/var/etcd/ssl/client/client/tls.crt --key=/var/etcd/ssl/client/client/tls.key get %s-%d", etcdName, etcdClientPort, keyPrefix, suffix) - stdout, stderr, err = executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) + cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://%s-client.shoot.svc:%d --cacert /var/etcd/ssl/ca/ca.crt --cert=/var/etcd/ssl/client/tls.crt --key=/var/etcd/ssl/client/tls.key get %s-%d", etcdName, etcdClientPort, keyPrefix, suffix) + stdout, stderr, err = executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stderr != "" { return "", "", fmt.Errorf("failed to get %s-%d: stdout: %s; stderr: %s; err: %v", keyPrefix, suffix, stdout, stderr, err) } @@ -724,7 +724,7 @@ func getEtcdKey(ctx context.Context, kubeconfigPath, namespace, etcdName, podNam return strings.TrimSpace(splits[0]), strings.TrimSpace(splits[1]), nil } -func getEtcdKeys(ctx context.Context, logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, start, end int) (map[string]string, error) { +func getEtcdKeys(logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, start, end int) (map[string]string, error) { var ( key string val string @@ -733,7 +733,7 @@ func getEtcdKeys(ctx context.Context, logger logr.Logger, kubeconfigPath, namesp err error ) for i := start; i <= end; { - key, val, err = getEtcdKey(ctx, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, i) + key, val, err = getEtcdKey(kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, i) if err != nil { logger.Info("failed to get key. Retrying...", "key", fmt.Sprintf("%s-%d", keyPrefix, i)) retries++ @@ -751,7 +751,7 @@ func getEtcdKeys(ctx context.Context, logger logr.Logger, kubeconfigPath, namesp return keyValueMap, nil } -func triggerOnDemandSnapshot(ctx context.Context, kubeconfigPath, namespace, etcdName, podName, containerName string, port int, snapshotKind string) (*brtypes.Snapshot, error) { +func triggerOnDemandSnapshot(kubeconfigPath, namespace, etcdName, podName, containerName string, port int, snapshotKind string) (*brtypes.Snapshot, error) { var ( snapshot *brtypes.Snapshot snapKind string @@ -766,7 +766,7 @@ func triggerOnDemandSnapshot(ctx context.Context, kubeconfigPath, namespace, etc return nil, fmt.Errorf("invalid snapshotKind %s", snapshotKind) } cmd := fmt.Sprintf("curl https://%s-client.shoot.svc:%d/snapshot/%s -k -s", etcdName, port, snapKind) - stdout, stderr, err := executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err := executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stdout == "" { return nil, fmt.Errorf("failed to trigger on-demand %s snapshot for %s: stdout: %s; stderr: %s; err: %v", snapKind, podName, stdout, stderr, err) } @@ -777,10 +777,10 @@ func triggerOnDemandSnapshot(ctx context.Context, kubeconfigPath, namespace, etc return snapshot, nil } -func getLatestSnapshots(ctx context.Context, kubeconfigPath, namespace, etcdName, podName, containerName string, port int) (*LatestSnapshots, error) { +func getLatestSnapshots(kubeconfigPath, namespace, etcdName, podName, containerName string, port int) (*LatestSnapshots, error) { var latestSnapshots *LatestSnapshots cmd := fmt.Sprintf("curl https://%s-client.shoot.svc:%d/snapshot/latest -k -s", etcdName, port) - stdout, stderr, err := executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err := executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stdout == "" { return nil, fmt.Errorf("failed to fetch latest snapshots taken for %s: stdout: %s; stderr: %s; err: %v", podName, stdout, stderr, err) } @@ -796,9 +796,9 @@ func getLatestSnapshots(ctx context.Context, kubeconfigPath, namespace, etcdName return latestSnapshots, nil } -func deleteDir(ctx context.Context, kubeconfigPath, namespace, podName, containerName string, dirPath string) error { +func deleteDir(kubeconfigPath, namespace, podName, containerName string, dirPath string) error { cmd := fmt.Sprintf("rm -rf %s", dirPath) - stdout, stderr, err := executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err := executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stdout != "" { return fmt.Errorf("failed to delete directory %s for %s: stdout: %s; stderr: %s; err: %v", dirPath, podName, stdout, stderr, err) } @@ -841,7 +841,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(416), + DefaultMode: pointer.Int32(0640), }, }, }, @@ -849,8 +849,8 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon Name: "client-url-etcd-server-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tls.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(416), + SecretName: tls.ServerTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -858,8 +858,8 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon Name: "client-url-etcd-client-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tls.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(416), + SecretName: tls.ClientTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -874,10 +874,10 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon "echo '" + "failed=0 ; threshold=2 ; " + "while [ $failed -lt $threshold ] ; do " + - "$(curl --cacert /var/etcd/ssl/client/ca/ca.crt --cert /var/etcd/ssl/client/client/tls.crt --key /var/etcd/ssl/client/client/tls.key https://" + etcdSvc + ":2379/health -s -f -o /dev/null ); " + + "$(curl --cacert /var/etcd/ssl/ca/ca.crt --cert /var/etcd/ssl/client/tls.crt --key /var/etcd/ssl/client/tls.key https://" + etcdSvc + ":2379/health -s -f -o /dev/null ); " + "if [ $? -gt 0 ] ; then let failed++; echo \"etcd is unhealthy and retrying\"; sleep 1; continue; fi ; " + "echo \"etcd is healthy\"; touch /tmp/healthy; let failed=0; " + - "sleep 1; done; echo \"etcd is unhealthy\"; exit 1;" + + "sleep 2; done; echo \"etcd is unhealthy\"; exit 1;" + "' > test.sh && sh test.sh", }, ReadinessProbe: &corev1.Probe{ @@ -909,15 +909,15 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon }, VolumeMounts: []corev1.VolumeMount{ { - MountPath: "/var/etcd/ssl/client/ca", + MountPath: "/var/etcd/ssl/ca", Name: "client-url-ca-etcd", }, { - MountPath: "/var/etcd/ssl/client/server", + MountPath: "/var/etcd/ssl/server", Name: "client-url-etcd-server-tls", }, { - MountPath: "/var/etcd/ssl/client/client", + MountPath: "/var/etcd/ssl/client", Name: "client-url-etcd-client-tls", ReadOnly: true, }, @@ -948,15 +948,15 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { Image: "nginx", VolumeMounts: []corev1.VolumeMount{ { - MountPath: "/var/etcd/ssl/client/ca", + MountPath: "/var/etcd/ssl/ca", Name: "client-url-ca-etcd", }, { - MountPath: "/var/etcd/ssl/client/server", + MountPath: "/var/etcd/ssl/server", Name: "client-url-etcd-server-tls", }, { - MountPath: "/var/etcd/ssl/client/client", + MountPath: "/var/etcd/ssl/client", Name: "client-url-etcd-client-tls", ReadOnly: true, }, @@ -973,7 +973,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(416), + DefaultMode: pointer.Int32(0640), }, }, }, @@ -981,8 +981,8 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { Name: "client-url-etcd-server-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(416), + SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, @@ -990,8 +990,8 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { Name: "client-url-etcd-client-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(416), + SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, + DefaultMode: pointer.Int32(0640), }, }, }, From 5bd2f1b5e670717d0797b047d4ae34e915911601 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 2 Apr 2024 14:04:38 +0530 Subject: [PATCH 125/235] fixed unit test for init container mount --- hack/e2e-test/infrastructure/kind/cluster.yaml | 2 +- internal/operator/statefulset/statefulset.go | 17 +++++------------ internal/operator/statefulset/stsmatcher.go | 2 +- internal/utils/lease.go | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/hack/e2e-test/infrastructure/kind/cluster.yaml b/hack/e2e-test/infrastructure/kind/cluster.yaml index 8ff2f09ce..90522d047 100644 --- a/hack/e2e-test/infrastructure/kind/cluster.yaml +++ b/hack/e2e-test/infrastructure/kind/cluster.yaml @@ -2,7 +2,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.27.1 + image: kindest/node:v1.29.2 # port forward 80 on the host to 80 on this node extraPortMappings: - containerPort: 4566 diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index e2e93bb1a..2af771f2f 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -158,12 +158,12 @@ func (r _resource) createOrPatch(ctx component.OperatorContext, etcd *druidv1alp } func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { - peerTLSEnabledForAllMembers, err := utils.IsPeerURLTLSEnabledForMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name, int(*existingSts.Spec.Replicas)) + peerTLSEnabledForMembers, err := utils.IsPeerURLTLSEnabledForMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name, *existingSts.Spec.Replicas) if err != nil { return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) } - if isPeerTLSChangedToEnabled(peerTLSEnabledForAllMembers, etcd) { + if isPeerTLSEnablementPending(peerTLSEnabledForMembers, etcd) { if !isStatefulSetPatchedWithPeerTLSVolMount(existingSts) { // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd component due to // enabling of TLS for peer communication. It preserves the current STS replicas. @@ -173,14 +173,7 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru } else { ctx.Logger.Info("Secret volume mounts to enable Peer URL TLS have already been mounted. Skipping patching StatefulSet with secret volume mounts.") } - // check again if peer TLS has been enabled for all members. If not then force a requeue of the reconcile request. - peerTLSEnabledForAllMembers, err = utils.IsPeerURLTLSEnabledForMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name, int(*existingSts.Spec.Replicas)) - if err != nil { - return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) - } - if !peerTLSEnabledForAllMembers { - return fmt.Errorf("peer URL TLS not enabled for all members for etcd: %v, requeuing reconcile request", etcd.GetNamespaceName()) - } + return fmt.Errorf("peer URL TLS not enabled for #%d members for etcd: %v, requeuing reconcile request", *existingSts.Spec.Replicas, etcd.GetNamespaceName()) } ctx.Logger.Info("Peer URL TLS has been enabled for all members") return nil @@ -200,8 +193,8 @@ func isStatefulSetPatchedWithPeerTLSVolMount(existingSts *appsv1.StatefulSet) bo return peerURLCAEtcdVolPresent && peerURLEtcdServerTLSVolPresent } -// isPeerTLSChangedToEnabled checks if the Peer TLS setting has changed to enabled -func isPeerTLSChangedToEnabled(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { +// isPeerTLSEnablementPending checks if the peer URL TLS has been enabled for the etcd, but it has not yet reflected in all etcd members. +func isPeerTLSEnablementPending(peerTLSEnabledStatusFromMembers bool, etcd *druidv1alpha1.Etcd) bool { return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index c424261cc..1c9db1b0c 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -182,7 +182,7 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Command": HaveExactElements("sh", "-c", "--"), "Args": HaveExactElements(fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *s.etcd.Spec.Backup.Store.Container)), - "VolumeMounts": s.matchBackupRestoreContainerVolMounts(), + "VolumeMounts": ConsistOf(s.getEtcdBackupVolumeMountMatcher()), "SecurityContext": matchPodInitContainerSecurityContext(), }) initContainerMatcherElements[common.ChangeBackupBucketPermissionsInitContainerName] = changeBackupBucketPermissionsMatcher diff --git a/internal/utils/lease.go b/internal/utils/lease.go index b352d77bf..ef766d4a4 100644 --- a/internal/utils/lease.go +++ b/internal/utils/lease.go @@ -20,7 +20,7 @@ import ( const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" // IsPeerURLTLSEnabledForMembers checks if TLS has been enabled for all existing members of an etcd cluster identified by etcdName and in the provided namespace. -func IsPeerURLTLSEnabledForMembers(ctx context.Context, cl client.Client, logger logr.Logger, namespace, etcdName string, numReplicas int) (bool, error) { +func IsPeerURLTLSEnabledForMembers(ctx context.Context, cl client.Client, logger logr.Logger, namespace, etcdName string, numReplicas int32) (bool, error) { leaseList := &coordinationv1.LeaseList{} if err := cl.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(map[string]string{ druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, From b41558a776149ee7d456ab83e76804586e6181b5 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 2 Apr 2024 14:32:02 +0530 Subject: [PATCH 126/235] Run `make revendor` --- .ci/hack/component_descriptor | 0 .ci/hack/prepare_release | 0 .ci/hack/set_dependency_version | 0 vendor/cloud.google.com/go/.gitignore | 12 + .../.release-please-manifest-submodules.json | 113 + .../go/.release-please-manifest.json | 3 + vendor/cloud.google.com/go/CHANGES.md | 2424 +++++++++++++++++ vendor/cloud.google.com/go/CODE_OF_CONDUCT.md | 44 + vendor/cloud.google.com/go/CONTRIBUTING.md | 327 +++ vendor/cloud.google.com/go/LICENSE | 202 ++ vendor/cloud.google.com/go/README.md | 139 + vendor/cloud.google.com/go/RELEASING.md | 141 + vendor/cloud.google.com/go/SECURITY.md | 7 + vendor/cloud.google.com/go/compute/LICENSE | 202 ++ .../go/compute/internal/version.go | 18 + .../go/compute/metadata/CHANGES.md | 5 + .../go/compute/metadata/LICENSE | 202 ++ .../go/compute/metadata/README.md | 27 + .../go/compute/metadata/metadata.go | 542 ++++ .../go/compute/metadata/retry.go | 114 + .../go/compute/metadata/retry_linux.go | 26 + .../go/compute/metadata/tidyfix.go | 23 + vendor/cloud.google.com/go/doc.go | 248 ++ vendor/cloud.google.com/go/iam/CHANGES.md | 62 + vendor/cloud.google.com/go/iam/LICENSE | 202 ++ vendor/cloud.google.com/go/iam/README.md | 40 + vendor/cloud.google.com/go/iam/iam.go | 387 +++ .../go/internal/.repo-metadata-full.json | 1946 +++++++++++++ vendor/cloud.google.com/go/internal/README.md | 18 + .../cloud.google.com/go/internal/annotate.go | 55 + .../go/internal/optional/optional.go | 108 + vendor/cloud.google.com/go/internal/retry.go | 85 + .../go/internal/trace/trace.go | 111 + .../go/internal/version/update_version.sh | 19 + .../go/internal/version/version.go | 71 + vendor/cloud.google.com/go/migration.md | 50 + ...elease-please-config-yoshi-submodules.json | 341 +++ .../go/release-please-config.json | 11 + vendor/cloud.google.com/go/testing.md | 236 ++ 39 files changed, 8561 insertions(+) mode change 100644 => 100755 .ci/hack/component_descriptor mode change 100644 => 100755 .ci/hack/prepare_release mode change 100644 => 100755 .ci/hack/set_dependency_version create mode 100644 vendor/cloud.google.com/go/.gitignore create mode 100644 vendor/cloud.google.com/go/.release-please-manifest-submodules.json create mode 100644 vendor/cloud.google.com/go/.release-please-manifest.json create mode 100644 vendor/cloud.google.com/go/CHANGES.md create mode 100644 vendor/cloud.google.com/go/CODE_OF_CONDUCT.md create mode 100644 vendor/cloud.google.com/go/CONTRIBUTING.md create mode 100644 vendor/cloud.google.com/go/LICENSE create mode 100644 vendor/cloud.google.com/go/README.md create mode 100644 vendor/cloud.google.com/go/RELEASING.md create mode 100644 vendor/cloud.google.com/go/SECURITY.md create mode 100644 vendor/cloud.google.com/go/compute/LICENSE create mode 100644 vendor/cloud.google.com/go/compute/internal/version.go create mode 100644 vendor/cloud.google.com/go/compute/metadata/CHANGES.md create mode 100644 vendor/cloud.google.com/go/compute/metadata/LICENSE create mode 100644 vendor/cloud.google.com/go/compute/metadata/README.md create mode 100644 vendor/cloud.google.com/go/compute/metadata/metadata.go create mode 100644 vendor/cloud.google.com/go/compute/metadata/retry.go create mode 100644 vendor/cloud.google.com/go/compute/metadata/retry_linux.go create mode 100644 vendor/cloud.google.com/go/compute/metadata/tidyfix.go create mode 100644 vendor/cloud.google.com/go/doc.go create mode 100644 vendor/cloud.google.com/go/iam/CHANGES.md create mode 100644 vendor/cloud.google.com/go/iam/LICENSE create mode 100644 vendor/cloud.google.com/go/iam/README.md create mode 100644 vendor/cloud.google.com/go/iam/iam.go create mode 100644 vendor/cloud.google.com/go/internal/.repo-metadata-full.json create mode 100644 vendor/cloud.google.com/go/internal/README.md create mode 100644 vendor/cloud.google.com/go/internal/annotate.go create mode 100644 vendor/cloud.google.com/go/internal/optional/optional.go create mode 100644 vendor/cloud.google.com/go/internal/retry.go create mode 100644 vendor/cloud.google.com/go/internal/trace/trace.go create mode 100644 vendor/cloud.google.com/go/internal/version/update_version.sh create mode 100644 vendor/cloud.google.com/go/internal/version/version.go create mode 100644 vendor/cloud.google.com/go/migration.md create mode 100644 vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json create mode 100644 vendor/cloud.google.com/go/release-please-config.json create mode 100644 vendor/cloud.google.com/go/testing.md diff --git a/.ci/hack/component_descriptor b/.ci/hack/component_descriptor old mode 100644 new mode 100755 diff --git a/.ci/hack/prepare_release b/.ci/hack/prepare_release old mode 100644 new mode 100755 diff --git a/.ci/hack/set_dependency_version b/.ci/hack/set_dependency_version old mode 100644 new mode 100755 diff --git a/vendor/cloud.google.com/go/.gitignore b/vendor/cloud.google.com/go/.gitignore new file mode 100644 index 000000000..cc7e53b46 --- /dev/null +++ b/vendor/cloud.google.com/go/.gitignore @@ -0,0 +1,12 @@ +# Editors +.idea +.vscode +*.swp +.history + +# Test files +*.test +coverage.txt + +# Other +.DS_Store diff --git a/vendor/cloud.google.com/go/.release-please-manifest-submodules.json b/vendor/cloud.google.com/go/.release-please-manifest-submodules.json new file mode 100644 index 000000000..355c70d1a --- /dev/null +++ b/vendor/cloud.google.com/go/.release-please-manifest-submodules.json @@ -0,0 +1,113 @@ +{ + "accessapproval": "1.4.0", + "accesscontextmanager": "1.3.0", + "aiplatform": "1.24.0", + "analytics": "0.12.0", + "apigateway": "1.3.0", + "apigeeconnect": "1.3.0", + "apigeeregistry": "0.2.1", + "apikeys": "0.2.0", + "appengine": "1.4.0", + "area120": "0.6.0", + "artifactregistry": "1.8.0", + "asset": "1.9.0", + "assuredworkloads": "1.8.0", + "automl": "1.7.0", + "baremetalsolution": "0.3.0", + "batch": "0.3.0", + "beyondcorp": "0.2.0", + "billing": "1.6.0", + "binaryauthorization": "1.3.0", + "certificatemanager": "1.3.0", + "channel": "1.8.0", + "cloudbuild": "1.3.0", + "clouddms": "1.3.0", + "cloudtasks": "1.7.0", + "compute": "1.12.1", + "compute/metadata": "0.1.1", + "contactcenterinsights": "1.3.0", + "container": "1.6.0", + "containeranalysis": "0.6.0", + "datacatalog": "1.7.0", + "dataflow": "0.7.0", + "dataform": "0.5.0", + "datafusion": "1.4.0", + "datalabeling": "0.6.0", + "dataplex": "1.3.0", + "dataproc": "1.7.0", + "dataqna": "0.6.0", + "datastream": "1.4.0", + "deploy": "1.4.0", + "dialogflow": "1.18.0", + "dlp": "1.6.0", + "documentai": "1.9.0", + "domains": "0.7.0", + "edgecontainer": "0.2.0", + "essentialcontacts": "1.3.0", + "eventarc": "1.7.0", + "filestore": "1.3.0", + "functions": "1.8.0", + "gaming": "1.7.0", + "gkebackup": "0.2.0", + "gkeconnect": "0.6.0", + "gkehub": "0.10.0", + "gkemulticloud": "0.3.0", + "grafeas": "0.2.0", + "gsuiteaddons": "1.3.0", + "iam": "0.6.0", + "iap": "1.4.0", + "ids": "1.1.0", + "iot": "1.3.0", + "kms": "1.5.0", + "language": "1.7.0", + "lifesciences": "0.6.0", + "managedidentities": "1.3.0", + "mediatranslation": "0.6.0", + "memcache": "1.6.0", + "metastore": "1.7.0", + "monitoring": "1.7.0", + "networkconnectivity": "1.6.0", + "networkmanagement": "1.4.0", + "networksecurity": "0.6.0", + "notebooks": "1.4.0", + "optimization": "1.1.0", + "orchestration": "1.3.0", + "orgpolicy": "1.4.0", + "osconfig": "1.9.0", + "oslogin": "1.6.0", + "phishingprotection": "0.6.0", + "policytroubleshooter": "1.3.0", + "privatecatalog": "0.6.0", + "recaptchaenterprise/v2": "2.4.0", + "recommendationengine": "0.6.0", + "recommender": "1.7.0", + "redis": "1.9.0", + "resourcemanager": "1.3.0", + "resourcesettings": "1.3.0", + "retail": "1.10.0", + "run": "0.2.0", + "scheduler": "1.6.0", + "secretmanager": "1.8.0", + "security": "1.9.0", + "securitycenter": "1.15.0", + "servicecontrol": "1.4.0", + "servicedirectory": "1.6.0", + "servicemanagement": "1.4.0", + "serviceusage": "1.3.0", + "shell": "1.3.0", + "speech": "1.8.0", + "storagetransfer": "1.5.0", + "talent": "1.3.0", + "texttospeech": "1.4.0", + "tpu": "1.3.0", + "trace": "1.3.0", + "translate": "1.3.0", + "video": "1.8.0", + "videointelligence": "1.8.0", + "vision/v2": "2.4.0", + "vmmigration": "1.2.0", + "vpcaccess": "1.4.0", + "webrisk": "1.6.0", + "websecurityscanner": "1.3.0", + "workflows": "1.8.0" +} diff --git a/vendor/cloud.google.com/go/.release-please-manifest.json b/vendor/cloud.google.com/go/.release-please-manifest.json new file mode 100644 index 000000000..d9c938906 --- /dev/null +++ b/vendor/cloud.google.com/go/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.104.0" +} diff --git a/vendor/cloud.google.com/go/CHANGES.md b/vendor/cloud.google.com/go/CHANGES.md new file mode 100644 index 000000000..8d00d2e95 --- /dev/null +++ b/vendor/cloud.google.com/go/CHANGES.md @@ -0,0 +1,2424 @@ +# Changes + +## [0.104.0](https://github.com/googleapis/google-cloud-go/compare/v0.103.0...v0.104.0) (2022-08-24) + + +### Features + +* **godocfx:** add friendlyAPIName ([#6447](https://github.com/googleapis/google-cloud-go/issues/6447)) ([c6d3ba4](https://github.com/googleapis/google-cloud-go/commit/c6d3ba401b7b3ae9b710a8850c6ec5d49c4c1490)) + +## [0.103.0](https://github.com/googleapis/google-cloud-go/compare/v0.102.1...v0.103.0) (2022-06-29) + + +### Features + +* **privateca:** temporarily remove REGAPIC support ([199b725](https://github.com/googleapis/google-cloud-go/commit/199b7250f474b1a6f53dcf0aac0c2966f4987b68)) + +## [0.102.1](https://github.com/googleapis/google-cloud-go/compare/v0.102.0...v0.102.1) (2022-06-17) + + +### Bug Fixes + +* **longrunning:** regapic remove path params duped as query params ([#6183](https://github.com/googleapis/google-cloud-go/issues/6183)) ([c963be3](https://github.com/googleapis/google-cloud-go/commit/c963be301f074779e6bb8c897d8064fa076e9e35)) + +## [0.102.0](https://github.com/googleapis/google-cloud-go/compare/v0.101.1...v0.102.0) (2022-05-24) + + +### Features + +* **civil:** add Before and After methods to civil.Time ([#5703](https://github.com/googleapis/google-cloud-go/issues/5703)) ([7acaaaf](https://github.com/googleapis/google-cloud-go/commit/7acaaafef47668c3e8382b8bc03475598c3db187)) + +### [0.101.1](https://github.com/googleapis/google-cloud-go/compare/v0.101.0...v0.101.1) (2022-05-03) + + +### Bug Fixes + +* **internal/gapicgen:** properly update modules that have no gapic changes ([#5945](https://github.com/googleapis/google-cloud-go/issues/5945)) ([de2befc](https://github.com/googleapis/google-cloud-go/commit/de2befcaa2a886499db9da6d4d04d28398c8d44b)) + +## [0.101.0](https://github.com/googleapis/google-cloud-go/compare/v0.100.2...v0.101.0) (2022-04-20) + + +### Features + +* **all:** bump grpc dep ([#5481](https://github.com/googleapis/google-cloud-go/issues/5481)) ([b12964d](https://github.com/googleapis/google-cloud-go/commit/b12964df5c63c647aaf204e73cfcdfd379d19682)) +* **internal/gapicgen:** change versionClient for gapics ([#5687](https://github.com/googleapis/google-cloud-go/issues/5687)) ([55f0d92](https://github.com/googleapis/google-cloud-go/commit/55f0d92bf112f14b024b4ab0076c9875a17423c9)) + + +### Bug Fixes + +* **internal/gapicgen:** add generation of internal/version.go for new client modules ([#5726](https://github.com/googleapis/google-cloud-go/issues/5726)) ([341e0df](https://github.com/googleapis/google-cloud-go/commit/341e0df1e44480706180cc5b07c49b3cee904095)) +* **internal/gapicgen:** don't gen version files for longrunning and debugger ([#5698](https://github.com/googleapis/google-cloud-go/issues/5698)) ([3a81108](https://github.com/googleapis/google-cloud-go/commit/3a81108c74cd8864c56b8ab5939afd864db3c64b)) +* **internal/gapicgen:** don't try to make snippets for non-gapics ([#5919](https://github.com/googleapis/google-cloud-go/issues/5919)) ([c94dddc](https://github.com/googleapis/google-cloud-go/commit/c94dddc60ef83a0584ba8f7dd24589d9db971672)) +* **internal/gapicgen:** move breaking change indicator if present ([#5452](https://github.com/googleapis/google-cloud-go/issues/5452)) ([e712df5](https://github.com/googleapis/google-cloud-go/commit/e712df5ebb45598a1653081d7e11e578bad22ff8)) +* **internal/godocfx:** prevent errors for filtered mods ([#5485](https://github.com/googleapis/google-cloud-go/issues/5485)) ([6cb9b89](https://github.com/googleapis/google-cloud-go/commit/6cb9b89b2d654c695eab00d8fb375cce0cd6e059)) + +## [0.100.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.99.0...v0.100.0) (2022-01-04) + + +### Features + +* **analytics/admin:** add the `AcknowledgeUserDataCollection` operation which acknowledges the terms of user data collection for the specified property feat: add the new resource type `DataStream`, which is planned to eventually replace `WebDataStream`, `IosAppDataStream`, `AndroidAppDataStream` resources fix!: remove `GetEnhancedMeasurementSettings`, `UpdateEnhancedMeasurementSettingsRequest`, `UpdateEnhancedMeasurementSettingsRequest` operations from the API feat: add `CreateDataStream`, `DeleteDataStream`, `UpdateDataStream`, `ListDataStreams` operations to support the new `DataStream` resource feat: add `DISPLAY_VIDEO_360_ADVERTISER_LINK`, `DISPLAY_VIDEO_360_ADVERTISER_LINK_PROPOSAL` fields to `ChangeHistoryResourceType` enum feat: add the `account` field to the `Property` type docs: update the documentation with a new list of valid values for `UserLink.direct_roles` field ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) +* **assuredworkloads:** EU Regions and Support With Sovereign Controls ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) +* **dialogflow/cx:** added the display name of the current page in webhook requests ([e0833b2](https://www.github.com/googleapis/google-cloud-go/commit/e0833b2853834ba79fd20ca2ae9c613d585dd2a5)) +* **dialogflow/cx:** added the display name of the current page in webhook requests ([e0833b2](https://www.github.com/googleapis/google-cloud-go/commit/e0833b2853834ba79fd20ca2ae9c613d585dd2a5)) +* **dialogflow:** added export documentation method feat: added filter in list documentations request feat: added option to import custom metadata from Google Cloud Storage in reload document request feat: added option to apply partial update to the smart messaging allowlist in reload document request feat: added filter in list knowledge bases request ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) +* **dialogflow:** removed OPTIONAL for speech model variant docs: added more docs for speech model variant and improved docs format for participant ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) +* **recaptchaenterprise:** add new reCAPTCHA Enterprise fraud annotations ([3dd34a2](https://www.github.com/googleapis/google-cloud-go/commit/3dd34a262edbff63b9aece8faddc2ff0d98ce42a)) + + +### Bug Fixes + +* **artifactregistry:** fix resource pattern ID segment name ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) +* **compute:** add parameter in compute bazel rules ([#692](https://www.github.com/googleapis/google-cloud-go/issues/692)) ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) +* **profiler:** refine regular expression for parsing backoff duration in E2E tests ([#5229](https://www.github.com/googleapis/google-cloud-go/issues/5229)) ([4438aeb](https://www.github.com/googleapis/google-cloud-go/commit/4438aebca2ec01d4dbf22287aa651937a381e043)) +* **profiler:** remove certificate expiration workaround ([#5222](https://www.github.com/googleapis/google-cloud-go/issues/5222)) ([2da36c9](https://www.github.com/googleapis/google-cloud-go/commit/2da36c95f44d5f88fd93cd949ab78823cea74fe7)) + +## [0.99.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.98.0...v0.99.0) (2021-12-06) + + +### Features + +* **dialogflow/cx:** added `TelephonyTransferCall` in response message ([fe27098](https://www.github.com/googleapis/google-cloud-go/commit/fe27098e5d429911428821ded57384353e699774)) + +## [0.98.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.97.0...v0.98.0) (2021-12-03) + + +### Features + +* **aiplatform:** add enable_private_service_connect field to Endpoint feat: add id field to DeployedModel feat: add service_attachment field to PrivateEndpoints feat: add endpoint_id to CreateEndpointRequest and method signature to CreateEndpoint feat: add method signature to CreateFeatureStore, CreateEntityType, CreateFeature feat: add network and enable_private_service_connect to IndexEndpoint feat: add service_attachment to IndexPrivateEndpoints feat: add stratified_split field to training_pipeline InputDataConfig ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) +* **aiplatform:** add featurestore service to aiplatform v1 feat: add metadata service to aiplatform v1 ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) +* **aiplatform:** Adds support for `google.protobuf.Value` pipeline parameters in the `parameter_values` field ([88a1cdb](https://www.github.com/googleapis/google-cloud-go/commit/88a1cdbef3cc337354a61bc9276725bfb9a686d8)) +* **aiplatform:** Tensorboard v1 protos release feat:Exposing a field for v1 CustomJob-Tensorboard integration. ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) +* **binaryauthorization:** add new admission rule types to Policy feat: update SignatureAlgorithm enum to match algorithm names in KMS feat: add SystemPolicyV1Beta1 service ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **channel:** add resource type to ChannelPartnerLink ([c206948](https://www.github.com/googleapis/google-cloud-go/commit/c2069487f6af5bcb37d519afeb60e312e35e67d5)) +* **cloudtasks:** add C++ rules for Cloud Tasks ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) +* **compute:** Move compute.v1 from googleapis-discovery to googleapis ([#675](https://www.github.com/googleapis/google-cloud-go/issues/675)) ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **compute:** Switch to string enums for compute ([#685](https://www.github.com/googleapis/google-cloud-go/issues/685)) ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) +* **contactcenterinsights:** Add ability to update phrase matchers feat: Add issue model stats to time series feat: Add display name to issue model stats ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **contactcenterinsights:** Add WriteDisposition to BigQuery Export API ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) +* **contactcenterinsights:** deprecate issue_matches docs: if conversation medium is unspecified, it will default to PHONE_CALL ([1a0720f](https://www.github.com/googleapis/google-cloud-go/commit/1a0720f2f33bb14617f5c6a524946a93209e1266)) +* **contactcenterinsights:** new feature flag disable_issue_modeling docs: fixed formatting issues in the reference documentation ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) +* **contactcenterinsights:** remove feature flag disable_issue_modeling ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) +* **datacatalog:** Added BigQueryDateShardedSpec.latest_shard_resource field feat: Added SearchCatalogResult.display_name field feat: Added SearchCatalogResult.description field ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **dataproc:** add Dataproc Serverless for Spark Batches API ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) +* **dataproc:** Add support for dataproc BatchController service ([8519b94](https://www.github.com/googleapis/google-cloud-go/commit/8519b948fee5dc82d39300c4d96e92c85fe78fe6)) +* **dialogflow/cx:** added API for changelogs docs: clarified semantic of the streaming APIs ([587bba5](https://www.github.com/googleapis/google-cloud-go/commit/587bba5ad792a92f252107aa38c6af50fb09fb58)) +* **dialogflow/cx:** added API for changelogs docs: clarified semantic of the streaming APIs ([587bba5](https://www.github.com/googleapis/google-cloud-go/commit/587bba5ad792a92f252107aa38c6af50fb09fb58)) +* **dialogflow/cx:** added support for comparing between versions docs: clarified security settings API reference ([83b941c](https://www.github.com/googleapis/google-cloud-go/commit/83b941c0983e44fdd18ceee8c6f3e91219d72ad1)) +* **dialogflow/cx:** added support for Deployments with ListDeployments and GetDeployment apis feat: added support for DeployFlow api under Environments feat: added support for TestCasesConfig under Environment docs: added long running operation explanation for several apis fix!: marked resource name of security setting as not-required ([8c5c6cf](https://www.github.com/googleapis/google-cloud-go/commit/8c5c6cf9df046b67998a8608d05595bd9e34feb0)) +* **dialogflow/cx:** allow setting custom CA for generic webhooks and release CompareVersions API docs: clarify DLP template reader usage ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) +* **dialogflow:** added support to configure security settings, language code and time zone on conversation profile ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **dialogflow:** support document metadata filter in article suggestion and smart reply model in human agent assistant ([e33350c](https://www.github.com/googleapis/google-cloud-go/commit/e33350cfcabcddcda1a90069383d39c68deb977a)) +* **dlp:** added deidentify replacement dictionaries feat: added field for BigQuery inspect template inclusion lists feat: added field to support infotype versioning ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) +* **domains:** added library for Cloud Domains v1 API. Also added methods for the transfer-in flow docs: improved API comments ([8519b94](https://www.github.com/googleapis/google-cloud-go/commit/8519b948fee5dc82d39300c4d96e92c85fe78fe6)) +* **functions:** Secret Manager integration fields 'secret_environment_variables' and 'secret_volumes' added feat: CMEK integration fields 'kms_key_name' and 'docker_repository' added ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **kms:** add OAEP+SHA1 to the list of supported algorithms ([8c5c6cf](https://www.github.com/googleapis/google-cloud-go/commit/8c5c6cf9df046b67998a8608d05595bd9e34feb0)) +* **kms:** add RPC retry information for MacSign, MacVerify, and GenerateRandomBytes Committer: [@bdhess](https://www.github.com/bdhess) ([1a0720f](https://www.github.com/googleapis/google-cloud-go/commit/1a0720f2f33bb14617f5c6a524946a93209e1266)) +* **kms:** add support for Raw PKCS[#1](https://www.github.com/googleapis/google-cloud-go/issues/1) signing keys ([58bea89](https://www.github.com/googleapis/google-cloud-go/commit/58bea89a3d177d5c431ff19310794e3296253353)) +* **monitoring/apiv3:** add CreateServiceTimeSeries RPC ([9e41088](https://www.github.com/googleapis/google-cloud-go/commit/9e41088bb395fbae0e757738277d5c95fa2749c8)) +* **monitoring/dashboard:** Added support for auto-close configurations ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) +* **monitoring/metricsscope:** promote apiv1 to GA ([#5135](https://www.github.com/googleapis/google-cloud-go/issues/5135)) ([33c0f63](https://www.github.com/googleapis/google-cloud-go/commit/33c0f63e0e0ce69d9ef6e57b04d1b8cc10ed2b78)) +* **osconfig:** OSConfig: add OS policy assignment rpcs ([83b941c](https://www.github.com/googleapis/google-cloud-go/commit/83b941c0983e44fdd18ceee8c6f3e91219d72ad1)) +* **osconfig:** Update OSConfig API ([e33350c](https://www.github.com/googleapis/google-cloud-go/commit/e33350cfcabcddcda1a90069383d39c68deb977a)) +* **osconfig:** Update osconfig v1 and v1alpha RecurringSchedule.Frequency with DAILY frequency ([59e548a](https://www.github.com/googleapis/google-cloud-go/commit/59e548acc249c7bddd9c884c2af35d582a408c4d)) +* **recaptchaenterprise:** add reCAPTCHA Enterprise account defender API methods ([88a1cdb](https://www.github.com/googleapis/google-cloud-go/commit/88a1cdbef3cc337354a61bc9276725bfb9a686d8)) +* **redis:** [Cloud Memorystore for Redis] Support Multiple Read Replicas when creating Instance ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **redis:** [Cloud Memorystore for Redis] Support Multiple Read Replicas when creating Instance ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **security/privateca:** add IAMPolicy & Locations mix-in support ([1a0720f](https://www.github.com/googleapis/google-cloud-go/commit/1a0720f2f33bb14617f5c6a524946a93209e1266)) +* **securitycenter:** Added a new API method UpdateExternalSystem, which enables updating a finding w/ external system metadata. External systems are a child resource under finding, and are housed on the finding itself, and can also be filtered on in Notifications, the ListFindings and GroupFindings API ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) +* **securitycenter:** Added mute related APIs, proto messages and fields ([3e7185c](https://www.github.com/googleapis/google-cloud-go/commit/3e7185c241d97ee342f132ae04bc93bb79a8e897)) +* **securitycenter:** Added resource type and display_name field to the FindingResult, and supported them in the filter for ListFindings and GroupFindings. Also added display_name to the resource which is surfaced in NotificationMessage ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) +* **securitycenter:** Added vulnerability field to the finding feat: Added type field to the resource which is surfaced in NotificationMessage ([090cc3a](https://www.github.com/googleapis/google-cloud-go/commit/090cc3ae0f8747a14cc904fc6d429e2f5379bb03)) +* **servicecontrol:** add C++ rules for many Cloud services ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) +* **speech:** add result_end_time to SpeechRecognitionResult ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) +* **speech:** added alternative_language_codes to RecognitionConfig feat: WEBM_OPUS codec feat: SpeechAdaptation configuration feat: word confidence feat: spoken punctuation and spoken emojis feat: hint boost in SpeechContext ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) +* **texttospeech:** update v1 proto ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) +* **workflows/executions:** add a stack_trace field to the Error messages specifying where the error occured feat: add call_log_level field to Execution messages doc: clarify requirement to escape strings within JSON arguments ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) + + +### Bug Fixes + +* **accesscontextmanager:** nodejs package name access-context-manager ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) +* **aiplatform:** Remove invalid resource annotations ([587bba5](https://www.github.com/googleapis/google-cloud-go/commit/587bba5ad792a92f252107aa38c6af50fb09fb58)) +* **compute/metadata:** return an error when all retries have failed ([#5063](https://www.github.com/googleapis/google-cloud-go/issues/5063)) ([c792a0d](https://www.github.com/googleapis/google-cloud-go/commit/c792a0d13db019c9964efeee5c6bc85b07ca50fa)), refs [#5062](https://www.github.com/googleapis/google-cloud-go/issues/5062) +* **compute:** make parent_id fields required compute move and insert methods ([#686](https://www.github.com/googleapis/google-cloud-go/issues/686)) ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) +* **compute:** Move compute_small protos under its own directory ([#681](https://www.github.com/googleapis/google-cloud-go/issues/681)) ([3e7185c](https://www.github.com/googleapis/google-cloud-go/commit/3e7185c241d97ee342f132ae04bc93bb79a8e897)) +* **internal/gapicgen:** fix a compute filtering ([#5111](https://www.github.com/googleapis/google-cloud-go/issues/5111)) ([77aa19d](https://www.github.com/googleapis/google-cloud-go/commit/77aa19de7fc33a9e831e6b91bd324d6832b44d99)) +* **internal/godocfx:** only put TOC status on mod if all pkgs have same status ([#4974](https://www.github.com/googleapis/google-cloud-go/issues/4974)) ([309b59e](https://www.github.com/googleapis/google-cloud-go/commit/309b59e583d1bf0dd9ffe84223034eb8a2975d47)) +* **internal/godocfx:** replace * with HTML code ([#5049](https://www.github.com/googleapis/google-cloud-go/issues/5049)) ([a8f7c06](https://www.github.com/googleapis/google-cloud-go/commit/a8f7c066e8d97120ae4e12963e3c9acc8b8906c2)) +* **monitoring/apiv3:** Reintroduce deprecated field/enum for backward compatibility docs: Use absolute link targets in comments ([45fd259](https://www.github.com/googleapis/google-cloud-go/commit/45fd2594d99ef70c776df26866f0a3b537e7e69e)) +* **profiler:** workaround certificate expiration issue in integration tests ([#4955](https://www.github.com/googleapis/google-cloud-go/issues/4955)) ([de9e465](https://www.github.com/googleapis/google-cloud-go/commit/de9e465bea8cd0580c45e87d2cbc2b610615b363)) +* **security/privateca:** include mixin protos as input for mixin rpcs ([479c2f9](https://www.github.com/googleapis/google-cloud-go/commit/479c2f90d556a106b25ebcdb1539d231488182da)) +* **security/privateca:** repair service config to enable mixins ([83b941c](https://www.github.com/googleapis/google-cloud-go/commit/83b941c0983e44fdd18ceee8c6f3e91219d72ad1)) +* **video/transcoder:** update nodejs package name to video-transcoder ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) + +## [0.97.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.96.0...v0.97.0) (2021-09-29) + + +### Features + +* **internal** add Retry func to testutil from samples repository [#4902](https://github.com/googleapis/google-cloud-go/pull/4902) + +## [0.96.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.95.0...v0.96.0) (2021-09-28) + + +### Features + +* **civil:** add IsEmpty function to time, date and datetime ([#4728](https://www.github.com/googleapis/google-cloud-go/issues/4728)) ([88bfa64](https://www.github.com/googleapis/google-cloud-go/commit/88bfa64d6df2f3bb7d41e0b8f56717dd3de790e2)), refs [#4727](https://www.github.com/googleapis/google-cloud-go/issues/4727) +* **internal/godocfx:** detect preview versions ([#4899](https://www.github.com/googleapis/google-cloud-go/issues/4899)) ([9b60844](https://www.github.com/googleapis/google-cloud-go/commit/9b608445ce9ebabbc87a50e85ce6ef89125031d2)) +* **internal:** provide wrapping for retried errors ([#4797](https://www.github.com/googleapis/google-cloud-go/issues/4797)) ([ce5f4db](https://www.github.com/googleapis/google-cloud-go/commit/ce5f4dbab884e847a2d9f1f8f3fcfd7df19a505a)) + + +### Bug Fixes + +* **internal/gapicgen:** restore fmting proto files ([#4789](https://www.github.com/googleapis/google-cloud-go/issues/4789)) ([5606b54](https://www.github.com/googleapis/google-cloud-go/commit/5606b54b97bb675487c6c138a4081c827218f933)) +* **internal/trace:** use xerrors.As for trace ([#4813](https://www.github.com/googleapis/google-cloud-go/issues/4813)) ([05fe61c](https://www.github.com/googleapis/google-cloud-go/commit/05fe61c5aa4860bdebbbe3e91a9afaba16aa6184)) + +## [0.95.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.94.1...v0.95.0) (2021-09-21) + +### Bug Fixes + +* **internal/gapicgen:** add a temporary import ([#4756](https://www.github.com/googleapis/google-cloud-go/issues/4756)) ([4d9c046](https://www.github.com/googleapis/google-cloud-go/commit/4d9c046b66a2dc205e2c14b676995771301440da)) +* **compute/metadata:** remove heavy gax dependency ([#4784](https://www.github.com/googleapis/google-cloud-go/issues/4784)) ([ea00264](https://www.github.com/googleapis/google-cloud-go/commit/ea00264428137471805f2ec67f04f3a5a42928fa)) + +### [0.94.1](https://www.github.com/googleapis/google-cloud-go/compare/v0.94.0...v0.94.1) (2021-09-02) + + +### Bug Fixes + +* **compute/metadata:** fix retry logic to not panic on error ([#4714](https://www.github.com/googleapis/google-cloud-go/issues/4714)) ([75c63b9](https://www.github.com/googleapis/google-cloud-go/commit/75c63b94d2cf86606fffc3611f7e6150b667eedc)), refs [#4713](https://www.github.com/googleapis/google-cloud-go/issues/4713) + +## [0.94.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.92.0...v0.94.0) (2021-08-31) + + +### Features + +* **aiplatform:** add XAI, model monitoring, and index services to aiplatform v1 ([e385b40](https://www.github.com/googleapis/google-cloud-go/commit/e385b40a1e2ecf81f5fd0910de5c37275951f86b)) +* **analytics/admin:** add `GetDataRetentionSettings`, `UpdateDataRetentionSettings` methods to the API ([8467899](https://www.github.com/googleapis/google-cloud-go/commit/8467899ab6ebf0328c543bfb5fbcddeb2f53a082)) +* **asset:** Release of relationships in v1, Add content type Relationship to support relationship export Committer: lvv@ ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) +* **assuredworkloads:** Add Canada Regions And Support compliance regime ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) +* **cloudbuild/apiv1:** Add ability to configure BuildTriggers to create Builds that require approval before executing and ApproveBuild API to approve or reject pending Builds ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) +* **cloudbuild/apiv1:** add script field to BuildStep message ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) +* **cloudbuild/apiv1:** Update cloudbuild proto with the service_account for BYOSA Triggers. ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) +* **compute/metadata:** retry error when talking to metadata service ([#4648](https://www.github.com/googleapis/google-cloud-go/issues/4648)) ([81c6039](https://www.github.com/googleapis/google-cloud-go/commit/81c6039503121f8da3de4f4cd957b8488a3ef620)), refs [#4642](https://www.github.com/googleapis/google-cloud-go/issues/4642) +* **dataproc:** remove apiv1beta2 client ([#4682](https://www.github.com/googleapis/google-cloud-go/issues/4682)) ([2248554](https://www.github.com/googleapis/google-cloud-go/commit/22485541affb1251604df292670a20e794111d3e)) +* **gaming:** support version reporting API ([cd65cec](https://www.github.com/googleapis/google-cloud-go/commit/cd65cecf15c4a01648da7f8f4f4d497772961510)) +* **gkehub:** Add request_id under `DeleteMembershipRequest` and `UpdateMembershipRequest` ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) +* **internal/carver:** support carving batches ([#4623](https://www.github.com/googleapis/google-cloud-go/issues/4623)) ([2972d19](https://www.github.com/googleapis/google-cloud-go/commit/2972d194da19bedf16d76fda471c06a965cfdcd6)) +* **kms:** add support for Key Reimport ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) +* **metastore:** Added the Backup resource and Backup resource GetIamPolicy/SetIamPolicy to V1 feat: Added the RestoreService method to V1 ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) +* **monitoring/dashboard:** Added support for logs-based alerts: https://cloud.google.com/logging/docs/alerting/log-based-alerts feat: Added support for user-defined labels on cloud monitoring's Service and ServiceLevelObjective objects fix!: mark required fields in QueryTimeSeriesRequest as required ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) +* **osconfig:** Update osconfig v1 and v1alpha with WindowsApplication ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) +* **speech:** Add transcript normalization ([b31646d](https://www.github.com/googleapis/google-cloud-go/commit/b31646d1e12037731df4b5c0ba9f60b6434d7b9b)) +* **talent:** Add new commute methods in Search APIs feat: Add new histogram type 'publish_time_in_day' feat: Support filtering by requisitionId is ListJobs API ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) +* **translate:** added v3 proto for online/batch document translation and updated v3beta1 proto for format conversion ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) + + +### Bug Fixes + +* **datastream:** Change a few resource pattern variables from camelCase to snake_case ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) + +## [0.92.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.91.0...v0.92.0) (2021-08-16) + + +### Features + +* **all:** remove testing deps ([#4580](https://www.github.com/googleapis/google-cloud-go/issues/4580)) ([15c1eb9](https://www.github.com/googleapis/google-cloud-go/commit/15c1eb9730f0b514edb911161f9c59e8d790a5ec)), refs [#4061](https://www.github.com/googleapis/google-cloud-go/issues/4061) +* **internal/detect:** add helper to detect projectID from env ([#4582](https://www.github.com/googleapis/google-cloud-go/issues/4582)) ([cc65d94](https://www.github.com/googleapis/google-cloud-go/commit/cc65d945688ac446602bce6ef86a935714dfe2f8)), refs [#1294](https://www.github.com/googleapis/google-cloud-go/issues/1294) +* **spannertest:** Add validation of duplicated column names ([#4611](https://www.github.com/googleapis/google-cloud-go/issues/4611)) ([84f86a6](https://www.github.com/googleapis/google-cloud-go/commit/84f86a605c809ab36dd3cb4b3ab1df15a5302083)) + +## [0.91.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.90.0...v0.91.0) (2021-08-11) + + +### Features + +* **.github:** support dynamic submodule detection ([#4537](https://www.github.com/googleapis/google-cloud-go/issues/4537)) ([4374b90](https://www.github.com/googleapis/google-cloud-go/commit/4374b907e9f166da6bd23a8ef94399872b00afd6)) +* **dialogflow/cx:** add advanced settings for agent level feat: add rollout config, state and failure reason for experiment feat: add insights export settings for security setting feat: add language code for streaming recognition result and flow versions for query parameters docs: deprecate legacy logging settings ([ed73554](https://www.github.com/googleapis/google-cloud-go/commit/ed735541dc57d0681d84b46853393eac5f7ccec3)) +* **dialogflow/cx:** add advanced settings for agent level feat: add rollout config, state and failure reason for experiment feat: add insights export settings for security setting feat: add language code for streaming recognition result and flow versions for query parameters docs: deprecate legacy logging settings ([ed73554](https://www.github.com/googleapis/google-cloud-go/commit/ed735541dc57d0681d84b46853393eac5f7ccec3)) +* **dialogflow/cx:** added support for DLP templates; expose `Locations` service to get/list avaliable locations of Dialogflow products ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) +* **dialogflow/cx:** added support for DLP templates; expose `Locations` service to get/list avaliable locations of Dialogflow products docs: reorder some fields ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) +* **dialogflow:** expose `Locations` service to get/list avaliable locations of Dialogflow products; fixed some API annotations ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) +* **kms:** add support for HMAC, Variable Key Destruction, and GenerateRandom ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) +* **speech:** add total_billed_time response field ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) +* **video/transcoder:** Add video cropping feature feat: Add video padding feature feat: Add ttl_after_completion_days field to Job docs: Update proto documentation docs: Indicate v1beta1 deprecation ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) + + +### Bug Fixes + +* **functions:** Updating behavior of source_upload_url during Get/List function calls ([381a494](https://www.github.com/googleapis/google-cloud-go/commit/381a494c29da388977b0bdda2177058328cc4afe)) + +## [0.90.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.89.0...v0.90.0) (2021-08-03) + + +### ⚠ BREAKING CHANGES + +* **compute:** add pagination and an Operation wrapper (#4542) + +### Features + +* **compute:** add pagination and an Operation wrapper ([#4542](https://www.github.com/googleapis/google-cloud-go/issues/4542)) ([36f4649](https://www.github.com/googleapis/google-cloud-go/commit/36f46494111f6d16d103fb208d49616576dbf91e)) +* **internal/godocfx:** add status to packages and TOCs ([#4547](https://www.github.com/googleapis/google-cloud-go/issues/4547)) ([c6de69c](https://www.github.com/googleapis/google-cloud-go/commit/c6de69c710561bb2a40eff05417df4b9798c258a)) +* **internal/godocfx:** mark status of deprecated items ([#4525](https://www.github.com/googleapis/google-cloud-go/issues/4525)) ([d571c6f](https://www.github.com/googleapis/google-cloud-go/commit/d571c6f4337ec9c4807c230cd77f53b6e7db6437)) + + +### Bug Fixes + +* **internal/carver:** don't tag commits ([#4518](https://www.github.com/googleapis/google-cloud-go/issues/4518)) ([c355eb8](https://www.github.com/googleapis/google-cloud-go/commit/c355eb8ecb0bb1af0ccf55e6262ca8c0d5c7e352)) + +## [0.89.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.88.0...v0.89.0) (2021-07-29) + + +### Features + +* **assuredworkloads:** Add EU Regions And Support compliance regime ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **datacatalog:** Added support for BigQuery connections entries feat: Added support for BigQuery routines entries feat: Added usage_signal field feat: Added labels field feat: Added ReplaceTaxonomy in Policy Tag Manager Serialization API feat: Added support for public tag templates feat: Added support for rich text tags docs: Documentation improvements ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **datafusion:** start generating apiv1 ([e55a016](https://www.github.com/googleapis/google-cloud-go/commit/e55a01667afaf36ff70807d061ecafb61827ba97)) +* **iap:** start generating apiv1 ([e55a016](https://www.github.com/googleapis/google-cloud-go/commit/e55a01667afaf36ff70807d061ecafb61827ba97)) +* **internal/carver:** add tooling to help carve out sub-modules ([#4417](https://www.github.com/googleapis/google-cloud-go/issues/4417)) ([a7e28f2](https://www.github.com/googleapis/google-cloud-go/commit/a7e28f2557469562ae57e5174b41bdf8fce62b63)) +* **networkconnectivity:** Add files for Network Connectivity v1 API. ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **retail:** Add restricted Retail Search features for Retail API v2. ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **secretmanager:** In Secret Manager, users can now use filter to customize the output of ListSecrets/ListSecretVersions calls ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **securitycenter:** add finding_class and indicator fields in Finding ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **speech:** add total_billed_time response field. fix!: phrase_set_id is required field in CreatePhraseSetRequest. fix!: custom_class_id is required field in CreateCustomClassRequest. ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) +* **storagetransfer:** start generating apiv1 ([#4505](https://www.github.com/googleapis/google-cloud-go/issues/4505)) ([f2d531d](https://www.github.com/googleapis/google-cloud-go/commit/f2d531d2b519efa58e0f23a178bbebe675c203c3)) + + +### Bug Fixes + +* **internal/gapicgen:** exec Stdout already set ([#4509](https://www.github.com/googleapis/google-cloud-go/issues/4509)) ([41246e9](https://www.github.com/googleapis/google-cloud-go/commit/41246e900aaaea92a9f956e92956c40c86f4cb3a)) +* **internal/gapicgen:** tidy all after dep bump ([#4515](https://www.github.com/googleapis/google-cloud-go/issues/4515)) ([9401be5](https://www.github.com/googleapis/google-cloud-go/commit/9401be509c570c3c55694375065c84139e961857)), refs [#4434](https://www.github.com/googleapis/google-cloud-go/issues/4434) + +## [0.88.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.87.0...v0.88.0) (2021-07-22) + + +### ⚠ BREAKING CHANGES + +* **cloudbuild/apiv1:** Proto had a prior definitions of WorkerPool resources which were never supported. This change replaces those resources with definitions that are currently supported. + +### Features + +* **cloudbuild/apiv1:** add a WorkerPools API ([19ea3f8](https://www.github.com/googleapis/google-cloud-go/commit/19ea3f830212582bfee21d9e09f0034f9ce76547)) +* **cloudbuild/apiv1:** Implementation of Build Failure Info: - Added message FailureInfo field ([19ea3f8](https://www.github.com/googleapis/google-cloud-go/commit/19ea3f830212582bfee21d9e09f0034f9ce76547)) +* **osconfig/agentendpoint:** OSConfig AgentEndpoint: add basic os info to RegisterAgentRequest, add WindowsApplication type to Inventory ([8936bc3](https://www.github.com/googleapis/google-cloud-go/commit/8936bc3f2d0fb2f6514f6e019fa247b8f41bd43c)) +* **resourcesettings:** Publish Cloud ResourceSettings v1 API ([43ad3cb](https://www.github.com/googleapis/google-cloud-go/commit/43ad3cb7be981fff9dc5dcf4510f1cd7bea99957)) + + +### Bug Fixes + +* **internal/godocfx:** set exit code, print cmd output, no go get ... ([#4445](https://www.github.com/googleapis/google-cloud-go/issues/4445)) ([cc70f77](https://www.github.com/googleapis/google-cloud-go/commit/cc70f77ac279a62e24e1b07f6e53fd126b7286b0)) +* **internal:** detect module for properly generating docs URLs ([#4460](https://www.github.com/googleapis/google-cloud-go/issues/4460)) ([1eaba8b](https://www.github.com/googleapis/google-cloud-go/commit/1eaba8bd694f7552a8e3e09b4f164de8b6ca23f0)), refs [#4447](https://www.github.com/googleapis/google-cloud-go/issues/4447) +* **kms:** Updating WORKSPACE files to use the newest version of the Typescript generator. ([8936bc3](https://www.github.com/googleapis/google-cloud-go/commit/8936bc3f2d0fb2f6514f6e019fa247b8f41bd43c)) + +## [0.87.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.86.0...v0.87.0) (2021-07-13) + + +### Features + +* **container:** allow updating security group on existing clusters ([528ffc9](https://www.github.com/googleapis/google-cloud-go/commit/528ffc9bd63090129a8b1355cd31273f8c23e34c)) +* **monitoring/dashboard:** added validation only mode when writing dashboards feat: added alert chart widget ([652d7c2](https://www.github.com/googleapis/google-cloud-go/commit/652d7c277da2f6774729064ab65d557875c81567)) +* **networkmanagment:** start generating apiv1 ([907592c](https://www.github.com/googleapis/google-cloud-go/commit/907592c576abfc65c01bbcd30c1a6094916cdc06)) +* **secretmanager:** Tune Secret Manager auto retry parameters ([528ffc9](https://www.github.com/googleapis/google-cloud-go/commit/528ffc9bd63090129a8b1355cd31273f8c23e34c)) +* **video/transcoder:** start generating apiv1 ([907592c](https://www.github.com/googleapis/google-cloud-go/commit/907592c576abfc65c01bbcd30c1a6094916cdc06)) + + +### Bug Fixes + +* **compute:** properly generate PUT requests ([#4426](https://www.github.com/googleapis/google-cloud-go/issues/4426)) ([a7491a5](https://www.github.com/googleapis/google-cloud-go/commit/a7491a533e4ad75eb6d5f89718d4dafb0c5b4167)) +* **internal:** fix relative pathing for generator ([#4397](https://www.github.com/googleapis/google-cloud-go/issues/4397)) ([25e0eae](https://www.github.com/googleapis/google-cloud-go/commit/25e0eaecf9feb1caa97988c5398ac58f6ca17391)) + + +### Miscellaneous Chores + +* **all:** fix release version ([#4427](https://www.github.com/googleapis/google-cloud-go/issues/4427)) ([2c0d267](https://www.github.com/googleapis/google-cloud-go/commit/2c0d2673ccab7281b6432215ee8279f9efd04a15)) + +## [0.86.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.85.0...v0.86.0) (2021-07-01) + + +### Features + +* **bigquery managedwriter:** schema conversion support ([#4357](https://www.github.com/googleapis/google-cloud-go/issues/4357)) ([f2b20f4](https://www.github.com/googleapis/google-cloud-go/commit/f2b20f493e2ed5a883ce42fa65695c03c574feb5)) + +## [0.85.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.84.0...v0.85.0) (2021-06-30) + + +### Features + +* **dataflow:** start generating apiv1beta3 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) +* **datastream:** start generating apiv1alpha1 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) +* **dialogflow:** added Automated agent reply type and allow cancellation flag for partial response feature. ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) +* **documentai:** update document.proto, add the processor management methods. ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) +* **eventarc:** start generating apiv1 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) +* **gkehub:** added v1alpha messages and client for gkehub ([8fb4649](https://www.github.com/googleapis/google-cloud-go/commit/8fb464956f0ca51d30e8e14dc625ff9fa150c437)) +* **internal/godocfx:** add support for other modules ([#4290](https://www.github.com/googleapis/google-cloud-go/issues/4290)) ([d52bae6](https://www.github.com/googleapis/google-cloud-go/commit/d52bae6cd77474174192c46236d309bf967dfa00)) +* **internal/godocfx:** different metadata for different modules ([#4297](https://www.github.com/googleapis/google-cloud-go/issues/4297)) ([598f5b9](https://www.github.com/googleapis/google-cloud-go/commit/598f5b93778b2e2e75265ae54484dd54477433f5)) +* **internal:** add force option for regen ([#4310](https://www.github.com/googleapis/google-cloud-go/issues/4310)) ([de654eb](https://www.github.com/googleapis/google-cloud-go/commit/de654ebfcf23a53b4d1fee0aa48c73999a55c1a6)) +* **servicecontrol:** Added the gRPC service config for the Service Controller v1 API docs: Updated some comments. ([8fb4649](https://www.github.com/googleapis/google-cloud-go/commit/8fb464956f0ca51d30e8e14dc625ff9fa150c437)) +* **workflows/executions:** start generating apiv1 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) + + +### Bug Fixes + +* **internal:** add autogenerated header to snippets ([#4261](https://www.github.com/googleapis/google-cloud-go/issues/4261)) ([2220787](https://www.github.com/googleapis/google-cloud-go/commit/222078722c37c3fdadec7bbbe0bcf81edd105f1a)), refs [#4260](https://www.github.com/googleapis/google-cloud-go/issues/4260) +* **internal:** fix googleapis-disco regen ([#4354](https://www.github.com/googleapis/google-cloud-go/issues/4354)) ([aeea1ce](https://www.github.com/googleapis/google-cloud-go/commit/aeea1ce1e5dff3acdfe208932327b52c49851b41)) +* **kms:** replace IAMPolicy mixin in service config. ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) +* **security/privateca:** Fixed casing of the Ruby namespace ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) + +## [0.84.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.83.0...v0.84.0) (2021-06-09) + + +### Features + +* **aiplatform:** start generating apiv1 ([be1d729](https://www.github.com/googleapis/google-cloud-go/commit/be1d729fdaa18eb1c782f3b09a6bb8fd6b3a144c)) +* **apigeeconnect:** start generating abiv1 ([be1d729](https://www.github.com/googleapis/google-cloud-go/commit/be1d729fdaa18eb1c782f3b09a6bb8fd6b3a144c)) +* **dialogflow/cx:** support sentiment analysis in bot testing ([7a57aac](https://www.github.com/googleapis/google-cloud-go/commit/7a57aac996f2bae20ee6ddbd02ad9e56e380099b)) +* **dialogflow/cx:** support sentiment analysis in bot testing ([6ad2306](https://www.github.com/googleapis/google-cloud-go/commit/6ad2306f64710ce16059b464342dbc6a98d2d9c2)) +* **documentai:** Move CommonOperationMetadata into a separate proto file for potential reuse. ([9e80ea0](https://www.github.com/googleapis/google-cloud-go/commit/9e80ea0d053b06876418194f65a478045dc4fe6c)) +* **documentai:** Move CommonOperationMetadata into a separate proto file for potential reuse. ([18375e5](https://www.github.com/googleapis/google-cloud-go/commit/18375e50e8f16e63506129b8927a7b62f85e407b)) +* **gkeconnect/gateway:** start generating apiv1beta1 ([#4235](https://www.github.com/googleapis/google-cloud-go/issues/4235)) ([1c3e968](https://www.github.com/googleapis/google-cloud-go/commit/1c3e9689d78670a231a3660db00fd4fd8f5c6345)) +* **lifesciences:** strat generating apiv2beta ([be1d729](https://www.github.com/googleapis/google-cloud-go/commit/be1d729fdaa18eb1c782f3b09a6bb8fd6b3a144c)) +* **tpu:** start generating apiv1 ([#4199](https://www.github.com/googleapis/google-cloud-go/issues/4199)) ([cac48ea](https://www.github.com/googleapis/google-cloud-go/commit/cac48eab960cd34cc20732f6a1aeb93c540a036b)) + + +### Bug Fixes + +* **bttest:** fix race condition in SampleRowKeys ([#4207](https://www.github.com/googleapis/google-cloud-go/issues/4207)) ([5711fb1](https://www.github.com/googleapis/google-cloud-go/commit/5711fb10d25c458807598d736a232bb2210a047a)) +* **documentai:** Fix Ruby gem title of documentai v1 (package not currently published) ([9e80ea0](https://www.github.com/googleapis/google-cloud-go/commit/9e80ea0d053b06876418194f65a478045dc4fe6c)) + +## [0.83.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.82.0...v0.83.0) (2021-06-02) + + +### Features + +* **dialogflow:** added a field in the query result to indicate whether slot filling is cancelled. ([f9cda8f](https://www.github.com/googleapis/google-cloud-go/commit/f9cda8fb6c3d76a062affebe6649f0a43aeb96f3)) +* **essentialcontacts:** start generating apiv1 ([#4118](https://www.github.com/googleapis/google-cloud-go/issues/4118)) ([fe14afc](https://www.github.com/googleapis/google-cloud-go/commit/fe14afcf74e09089b22c4f5221cbe37046570fda)) +* **gsuiteaddons:** start generating apiv1 ([#4082](https://www.github.com/googleapis/google-cloud-go/issues/4082)) ([6de5c99](https://www.github.com/googleapis/google-cloud-go/commit/6de5c99173c4eeaf777af18c47522ca15637d232)) +* **osconfig:** OSConfig: add ExecResourceOutput and per step error message. ([f9cda8f](https://www.github.com/googleapis/google-cloud-go/commit/f9cda8fb6c3d76a062affebe6649f0a43aeb96f3)) +* **osconfig:** start generating apiv1alpha ([#4119](https://www.github.com/googleapis/google-cloud-go/issues/4119)) ([8ad471f](https://www.github.com/googleapis/google-cloud-go/commit/8ad471f26087ec076460df6dcf27769ffe1b8834)) +* **privatecatalog:** start generating apiv1beta1 ([500c1a6](https://www.github.com/googleapis/google-cloud-go/commit/500c1a6101f624cb6032f0ea16147645a02e7076)) +* **serviceusage:** start generating apiv1 ([#4120](https://www.github.com/googleapis/google-cloud-go/issues/4120)) ([e4531f9](https://www.github.com/googleapis/google-cloud-go/commit/e4531f93cfeb6388280bb253ef6eb231aba37098)) +* **shell:** start generating apiv1 ([500c1a6](https://www.github.com/googleapis/google-cloud-go/commit/500c1a6101f624cb6032f0ea16147645a02e7076)) +* **vpcaccess:** start generating apiv1 ([500c1a6](https://www.github.com/googleapis/google-cloud-go/commit/500c1a6101f624cb6032f0ea16147645a02e7076)) + +## [0.82.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.81.0...v0.82.0) (2021-05-17) + + +### Features + +* **billing/budgets:** Added support for configurable budget time period. fix: Updated some documentation links. ([83b1b3b](https://www.github.com/googleapis/google-cloud-go/commit/83b1b3b648c6d9225f07f00e8c0cdabc3d1fc1ab)) +* **billing/budgets:** Added support for configurable budget time period. fix: Updated some documentation links. ([83b1b3b](https://www.github.com/googleapis/google-cloud-go/commit/83b1b3b648c6d9225f07f00e8c0cdabc3d1fc1ab)) +* **cloudbuild/apiv1:** Add fields for Pub/Sub triggers ([8b4adbf](https://www.github.com/googleapis/google-cloud-go/commit/8b4adbf9815e1ec229dfbcfb9189d3ea63112e1b)) +* **cloudbuild/apiv1:** Implementation of Source Manifests: - Added message StorageSourceManifest as an option for the Source message - Added StorageSourceManifest field to the SourceProvenance message ([7fd2ccd](https://www.github.com/googleapis/google-cloud-go/commit/7fd2ccd26adec1468e15fe84bf75210255a9dfea)) +* **clouddms:** start generating apiv1 ([#4081](https://www.github.com/googleapis/google-cloud-go/issues/4081)) ([29df85c](https://www.github.com/googleapis/google-cloud-go/commit/29df85c40ab64d59e389a980c9ce550077839763)) +* **dataproc:** update the Dataproc V1 API client library ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) +* **dialogflow/cx:** add support for service directory webhooks ([7fd2ccd](https://www.github.com/googleapis/google-cloud-go/commit/7fd2ccd26adec1468e15fe84bf75210255a9dfea)) +* **dialogflow/cx:** add support for service directory webhooks ([7fd2ccd](https://www.github.com/googleapis/google-cloud-go/commit/7fd2ccd26adec1468e15fe84bf75210255a9dfea)) +* **dialogflow/cx:** support setting current_page to resume sessions; expose transition_route_groups in flows and language_code in webhook ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) +* **dialogflow/cx:** support setting current_page to resume sessions; expose transition_route_groups in flows and language_code in webhook ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) +* **dialogflow:** added more Environment RPCs feat: added Versions service feat: added Fulfillment service feat: added TextToSpeechSettings. feat: added location in some resource patterns. ([4f73dc1](https://www.github.com/googleapis/google-cloud-go/commit/4f73dc19c2e05ad6133a8eac3d62ddb522314540)) +* **documentai:** add confidence field to the PageAnchor.PageRef in document.proto. ([d089dda](https://www.github.com/googleapis/google-cloud-go/commit/d089dda0089acb9aaef9b3da40b219476af9fc06)) +* **documentai:** add confidence field to the PageAnchor.PageRef in document.proto. ([07fdcd1](https://www.github.com/googleapis/google-cloud-go/commit/07fdcd12499eac26f9b5fae01d6c1282c3e02b7c)) +* **internal/gapicgen:** only update relevant gapic files ([#4066](https://www.github.com/googleapis/google-cloud-go/issues/4066)) ([5948bee](https://www.github.com/googleapis/google-cloud-go/commit/5948beedbadd491601bdee6a006cf685e94a85f4)) +* **internal/gensnippets:** add license header and region tags ([#3924](https://www.github.com/googleapis/google-cloud-go/issues/3924)) ([e9ff7a0](https://www.github.com/googleapis/google-cloud-go/commit/e9ff7a0f9bb1cc67f5d0de47934811960429e72c)) +* **internal/gensnippets:** initial commit ([#3922](https://www.github.com/googleapis/google-cloud-go/issues/3922)) ([3fabef0](https://www.github.com/googleapis/google-cloud-go/commit/3fabef032388713f732ab4dbfc51624cdca0f481)) +* **internal:** auto-generate snippets ([#3949](https://www.github.com/googleapis/google-cloud-go/issues/3949)) ([b70e0fc](https://www.github.com/googleapis/google-cloud-go/commit/b70e0fccdc86813e0d97ff63b585822d4deafb38)) +* **internal:** generate region tags for snippets ([#3962](https://www.github.com/googleapis/google-cloud-go/issues/3962)) ([ef2b90e](https://www.github.com/googleapis/google-cloud-go/commit/ef2b90ea6d47e27744c98a1a9ae0c487c5051808)) +* **metastore:** start generateing apiv1 ([#4083](https://www.github.com/googleapis/google-cloud-go/issues/4083)) ([661610a](https://www.github.com/googleapis/google-cloud-go/commit/661610afa6a9113534884cafb138109536724310)) +* **security/privateca:** start generating apiv1 ([#4023](https://www.github.com/googleapis/google-cloud-go/issues/4023)) ([08aa83a](https://www.github.com/googleapis/google-cloud-go/commit/08aa83a5371bb6485bc3b19b3ed5300f807ce69f)) +* **securitycenter:** add canonical_name and folder fields ([5c5ca08](https://www.github.com/googleapis/google-cloud-go/commit/5c5ca08c637a23cfa3e3a051fea576e1feb324fd)) +* **securitycenter:** add canonical_name and folder fields ([5c5ca08](https://www.github.com/googleapis/google-cloud-go/commit/5c5ca08c637a23cfa3e3a051fea576e1feb324fd)) +* **speech:** add webm opus support. ([d089dda](https://www.github.com/googleapis/google-cloud-go/commit/d089dda0089acb9aaef9b3da40b219476af9fc06)) +* **speech:** Support for spoken punctuation and spoken emojis. ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) + + +### Bug Fixes + +* **binaryauthorization:** add Java options to Binaryauthorization protos ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) +* **internal/gapicgen:** filter out internal directory changes ([#4085](https://www.github.com/googleapis/google-cloud-go/issues/4085)) ([01473f6](https://www.github.com/googleapis/google-cloud-go/commit/01473f6d8db26c6e18969ace7f9e87c66e94ad9e)) +* **internal/gapicgen:** use correct region tags for gensnippets ([#4022](https://www.github.com/googleapis/google-cloud-go/issues/4022)) ([8ccd689](https://www.github.com/googleapis/google-cloud-go/commit/8ccd689cab08f016008ca06a939a4828817d4a25)) +* **internal/gensnippets:** run goimports ([#3931](https://www.github.com/googleapis/google-cloud-go/issues/3931)) ([10050f0](https://www.github.com/googleapis/google-cloud-go/commit/10050f05c20c226547d87c08168fa4bc551395c5)) +* **internal:** append a new line to comply with go fmt ([#4028](https://www.github.com/googleapis/google-cloud-go/issues/4028)) ([a297278](https://www.github.com/googleapis/google-cloud-go/commit/a2972783c4af806199d1c67c9f63ad9677f20f34)) +* **internal:** make sure formatting is run on snippets ([#4039](https://www.github.com/googleapis/google-cloud-go/issues/4039)) ([130dfc5](https://www.github.com/googleapis/google-cloud-go/commit/130dfc535396e98fc009585b0457e3bc48ead941)), refs [#4037](https://www.github.com/googleapis/google-cloud-go/issues/4037) +* **metastore:** increase metastore lro polling timeouts ([83b1b3b](https://www.github.com/googleapis/google-cloud-go/commit/83b1b3b648c6d9225f07f00e8c0cdabc3d1fc1ab)) + + +### Miscellaneous Chores + +* **all:** fix release version ([#4040](https://www.github.com/googleapis/google-cloud-go/issues/4040)) ([4c991a9](https://www.github.com/googleapis/google-cloud-go/commit/4c991a928665d9be93691decce0c653f430688b7)) + +## [0.81.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.80.0...v0.81.0) (2021-04-02) + + +### Features + +* **datacatalog:** Policy Tag Manager v1 API service feat: new RenameTagTemplateFieldEnumValue API feat: adding fully_qualified_name in lookup and search feat: added DATAPROC_METASTORE integrated system along with new entry types: DATABASE and SERVICE docs: Documentation improvements ([2b02a03](https://www.github.com/googleapis/google-cloud-go/commit/2b02a03ff9f78884da5a8e7b64a336014c61bde7)) +* **dialogflow/cx:** include original user query in WebhookRequest; add GetTextCaseresult API. doc: clarify resource format for session response. ([a0b1f6f](https://www.github.com/googleapis/google-cloud-go/commit/a0b1f6faae77d014fdee166ab018ddcd6f846ab4)) +* **dialogflow/cx:** include original user query in WebhookRequest; add GetTextCaseresult API. doc: clarify resource format for session response. ([b5b4da6](https://www.github.com/googleapis/google-cloud-go/commit/b5b4da6952922440d03051f629f3166f731dfaa3)) +* **dialogflow:** expose MP3_64_KBPS and MULAW for output audio encodings. ([b5b4da6](https://www.github.com/googleapis/google-cloud-go/commit/b5b4da6952922440d03051f629f3166f731dfaa3)) +* **secretmanager:** Rotation for Secrets ([2b02a03](https://www.github.com/googleapis/google-cloud-go/commit/2b02a03ff9f78884da5a8e7b64a336014c61bde7)) + + +### Bug Fixes + +* **internal/godocfx:** filter out non-Cloud ([#3878](https://www.github.com/googleapis/google-cloud-go/issues/3878)) ([625aef9](https://www.github.com/googleapis/google-cloud-go/commit/625aef9b47181cf627587cc9cde9e400713c6678)) + +## [0.80.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.79.0...v0.80.0) (2021-03-23) + + +### ⚠ BREAKING CHANGES + +* **all:** This is a breaking change in dialogflow + +### Features + +* **appengine:** added vm_liveness, search_api_available, network_settings, service_account, build_env_variables, kms_key_reference to v1 API ([fd04a55](https://www.github.com/googleapis/google-cloud-go/commit/fd04a552213f99619c714b5858548f61f4948493)) +* **assuredworkloads:** Add 'resource_settings' field to provide custom properties (ids) for the provisioned projects. ([ab4824a](https://www.github.com/googleapis/google-cloud-go/commit/ab4824a7914864228e59b244d6382de862139524)) +* **assuredworkloads:** add HIPAA and HITRUST compliance regimes ([ab4824a](https://www.github.com/googleapis/google-cloud-go/commit/ab4824a7914864228e59b244d6382de862139524)) +* **dialogflow/cx:** added fallback option when restoring an agent docs: clarified experiment length ([cd70aa9](https://www.github.com/googleapis/google-cloud-go/commit/cd70aa9cc1a5dccfe4e49d2d6ca6db2119553c86)) +* **dialogflow/cx:** start generating apiv3 ([#3850](https://www.github.com/googleapis/google-cloud-go/issues/3850)) ([febbdcf](https://www.github.com/googleapis/google-cloud-go/commit/febbdcf13fcea3f5d8186c3d3dface1c0d27ef9e)), refs [#3634](https://www.github.com/googleapis/google-cloud-go/issues/3634) +* **documentai:** add EVAL_SKIPPED value to the Provenance.OperationType enum in document.proto. ([cb43066](https://www.github.com/googleapis/google-cloud-go/commit/cb4306683926843f6e977f207fa6070bb9242a61)) +* **documentai:** start generating apiv1 ([#3853](https://www.github.com/googleapis/google-cloud-go/issues/3853)) ([d68e604](https://www.github.com/googleapis/google-cloud-go/commit/d68e604c953eea90489f6134e71849b24dd0fcbf)) +* **internal/godocfx:** add prettyprint class to code blocks ([#3819](https://www.github.com/googleapis/google-cloud-go/issues/3819)) ([6e49f21](https://www.github.com/googleapis/google-cloud-go/commit/6e49f2148b116ee439c8a882dcfeefb6e7647c57)) +* **internal/godocfx:** handle Markdown content ([#3816](https://www.github.com/googleapis/google-cloud-go/issues/3816)) ([56d5d0a](https://www.github.com/googleapis/google-cloud-go/commit/56d5d0a900197fb2de46120a0eda649f2c17448f)) +* **kms:** Add maxAttempts to retry policy for KMS gRPC service config feat: Add Bazel exports_files entry for KMS gRPC service config ([fd04a55](https://www.github.com/googleapis/google-cloud-go/commit/fd04a552213f99619c714b5858548f61f4948493)) +* **resourcesettings:** start generating apiv1 ([#3854](https://www.github.com/googleapis/google-cloud-go/issues/3854)) ([3b288b4](https://www.github.com/googleapis/google-cloud-go/commit/3b288b4fa593c6cb418f696b5b26768967c20b9e)) +* **speech:** Support output transcript to GCS for LongRunningRecognize. ([fd04a55](https://www.github.com/googleapis/google-cloud-go/commit/fd04a552213f99619c714b5858548f61f4948493)) +* **speech:** Support output transcript to GCS for LongRunningRecognize. ([cd70aa9](https://www.github.com/googleapis/google-cloud-go/commit/cd70aa9cc1a5dccfe4e49d2d6ca6db2119553c86)) +* **speech:** Support output transcript to GCS for LongRunningRecognize. ([35a8706](https://www.github.com/googleapis/google-cloud-go/commit/35a870662df8bf63c4ec10a0233d1d7a708007ee)) + + +### Miscellaneous Chores + +* **all:** auto-regenerate gapics ([#3837](https://www.github.com/googleapis/google-cloud-go/issues/3837)) ([ab4824a](https://www.github.com/googleapis/google-cloud-go/commit/ab4824a7914864228e59b244d6382de862139524)) + +## [0.79.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.78.0...v0.79.0) (2021-03-10) + + +### Features + +* **apigateway:** start generating apiv1 ([#3726](https://www.github.com/googleapis/google-cloud-go/issues/3726)) ([66046da](https://www.github.com/googleapis/google-cloud-go/commit/66046da2a4be5971ce2655dc6a5e1fadb08c3d1f)) +* **channel:** addition of billing_account field on Plan. docs: clarification that valid address lines are required for all customers. ([d4246aa](https://www.github.com/googleapis/google-cloud-go/commit/d4246aad4da3c3ef12350385f229bb908e3fb215)) +* **dialogflow/cx:** allow to disable webhook invocation per request ([d4246aa](https://www.github.com/googleapis/google-cloud-go/commit/d4246aad4da3c3ef12350385f229bb908e3fb215)) +* **dialogflow/cx:** allow to disable webhook invocation per request ([44c6bf9](https://www.github.com/googleapis/google-cloud-go/commit/44c6bf986f39a3c9fddf46788ae63bfbb3739441)) +* **dialogflow:** Add CCAI API ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) +* **documentai:** remove the translation fields in document.proto. ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) +* **documentai:** Update documentai/v1beta3 protos: add support for boolean normalized value ([529925b](https://www.github.com/googleapis/google-cloud-go/commit/529925ba79f4d3191ef80a13e566d86210fe4d25)) +* **internal/godocfx:** keep some cross links on same domain ([#3767](https://www.github.com/googleapis/google-cloud-go/issues/3767)) ([77f76ed](https://www.github.com/googleapis/google-cloud-go/commit/77f76ed09cb07a090ba9054063a7c002a35bca4e)) +* **internal:** add ability to regenerate one module's docs ([#3777](https://www.github.com/googleapis/google-cloud-go/issues/3777)) ([dc15995](https://www.github.com/googleapis/google-cloud-go/commit/dc15995521bd065da4cfaae95642588919a8c548)) +* **metastore:** added support for release channels when creating service ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) +* **metastore:** Publish Dataproc Metastore v1alpha API ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) +* **metastore:** start generating apiv1alpha ([#3747](https://www.github.com/googleapis/google-cloud-go/issues/3747)) ([359312a](https://www.github.com/googleapis/google-cloud-go/commit/359312ad6d4f61fb341d41ffa35fc0634979e650)) +* **metastore:** start generating apiv1beta ([#3788](https://www.github.com/googleapis/google-cloud-go/issues/3788)) ([2977095](https://www.github.com/googleapis/google-cloud-go/commit/297709593ad32f234c0fbcfa228cffcfd3e591f4)) +* **secretmanager:** added topic field to Secret ([f1323b1](https://www.github.com/googleapis/google-cloud-go/commit/f1323b10a3c7cc1d215730cefd3062064ef54c01)) + + +### Bug Fixes + +* **analytics/admin:** add `https://www.googleapis.com/auth/analytics.edit` OAuth2 scope to the list of acceptable scopes for all read only methods of the Admin API docs: update the documentation of the `update_mask` field used by Update() methods ([f1323b1](https://www.github.com/googleapis/google-cloud-go/commit/f1323b10a3c7cc1d215730cefd3062064ef54c01)) +* **apigateway:** Provide resource definitions for service management and IAM resources ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) +* **functions:** Fix service namespace in grpc_service_config. ([7811a34](https://www.github.com/googleapis/google-cloud-go/commit/7811a34ef64d722480c640810251bb3a0d65d495)) +* **internal/godocfx:** prevent index out of bounds when pkg == mod ([#3768](https://www.github.com/googleapis/google-cloud-go/issues/3768)) ([3d80b4e](https://www.github.com/googleapis/google-cloud-go/commit/3d80b4e93b0f7e857d6e9681d8d6a429750ecf80)) +* **internal/godocfx:** use correct anchor links ([#3738](https://www.github.com/googleapis/google-cloud-go/issues/3738)) ([919039a](https://www.github.com/googleapis/google-cloud-go/commit/919039a01a006c41e720218bd55f83ce98a5edef)) +* **internal:** fix Bash syntax ([#3779](https://www.github.com/googleapis/google-cloud-go/issues/3779)) ([3dd245d](https://www.github.com/googleapis/google-cloud-go/commit/3dd245dbdbfa84f0bbe5a476412d8463fe3e700c)) +* **tables:** use area120tables_v1alpha1.yaml as api-service-config ([#3759](https://www.github.com/googleapis/google-cloud-go/issues/3759)) ([b130ec0](https://www.github.com/googleapis/google-cloud-go/commit/b130ec0aa946b1a1eaa4d5a7c33e72353ac1612e)) + +## [0.78.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.77.0...v0.78.0) (2021-02-22) + + +### Features + +* **area120/tables:** Added ListWorkspaces, GetWorkspace, BatchDeleteRows APIs. ([16597fa](https://www.github.com/googleapis/google-cloud-go/commit/16597fa1ce549053c7183e8456e23f554a5501de)) +* **area120/tables:** Added ListWorkspaces, GetWorkspace, BatchDeleteRows APIs. ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) +* **dialogflow:** add additional_bindings to Dialogflow v2 ListIntents API docs: update copyrights and session docs ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) +* **documentai:** Update documentai/v1beta3 protos ([613ced7](https://www.github.com/googleapis/google-cloud-go/commit/613ced702bbc82a154a4d3641b483f71c7cd1af4)) +* **gkehub:** Update Membership API v1beta1 proto ([613ced7](https://www.github.com/googleapis/google-cloud-go/commit/613ced702bbc82a154a4d3641b483f71c7cd1af4)) +* **servicecontrol:** Update the ruby_cloud_gapic_library rules for the libraries published to google-cloud-ruby to the form that works with build_gen (separate parameters for ruby_cloud_title and ruby_cloud_description). chore: Update Bazel-Ruby rules version. chore: Update build_gen version. ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) +* **speech:** Support Model Adaptation. ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) + + +### Bug Fixes + +* **dialogflow/cx:** RunTestCase http template. PHP REST client lib can be generated. feat: Support transition route group coverage for Test Cases. ([613ced7](https://www.github.com/googleapis/google-cloud-go/commit/613ced702bbc82a154a4d3641b483f71c7cd1af4)) +* **errorreporting:** Fixes ruby gem build ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) + +## [0.77.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.76.0...v0.77.0) (2021-02-16) + + +### Features + +* **channel:** Add Pub/Sub endpoints for Cloud Channel API. ([1aea7c8](https://www.github.com/googleapis/google-cloud-go/commit/1aea7c87d39eed87620b488ba0dd60b88ff26c04)) +* **dialogflow/cx:** supports SentimentAnalysisResult in webhook request docs: minor updates in wording ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) +* **errorreporting:** Make resolution status field available for error groups. Now callers can set the status of an error group by passing this to UpdateGroup. When not specified, it's treated like OPEN. feat: Make source location available for error groups created from GAE. ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) +* **errorreporting:** Make resolution status field available for error groups. Now callers can set the status of an error group by passing this to UpdateGroup. When not specified, it's treated like OPEN. feat: Make source location available for error groups created from GAE. ([f66114b](https://www.github.com/googleapis/google-cloud-go/commit/f66114bc7233ad06e18f38dd39497a74d85fdbd8)) +* **gkehub:** start generating apiv1beta1 ([#3698](https://www.github.com/googleapis/google-cloud-go/issues/3698)) ([8aed3bd](https://www.github.com/googleapis/google-cloud-go/commit/8aed3bd1bbbe983e4891c813e4c5dc9b3aa1b9b2)) +* **internal/docfx:** full cross reference linking ([#3656](https://www.github.com/googleapis/google-cloud-go/issues/3656)) ([fcb7318](https://www.github.com/googleapis/google-cloud-go/commit/fcb7318eb338bf3828ac831ed06ca630e1876418)) +* **memcache:** added ApplySoftwareUpdate API docs: various clarifications, new documentation for ApplySoftwareUpdate chore: update proto annotations ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) +* **networkconnectivity:** Add state field in resources docs: Minor changes ([0b4370a](https://www.github.com/googleapis/google-cloud-go/commit/0b4370a0d397913d932dbbdc2046a958dc3b836a)) +* **networkconnectivity:** Add state field in resources docs: Minor changes ([b4b5898](https://www.github.com/googleapis/google-cloud-go/commit/b4b58987368f80494bbc7f651f50e9123200fb3f)) +* **recommendationengine:** start generating apiv1beta1 ([#3686](https://www.github.com/googleapis/google-cloud-go/issues/3686)) ([8f4e130](https://www.github.com/googleapis/google-cloud-go/commit/8f4e13009444d88a5a56144129f055623a2205ac)) + + +### Bug Fixes + +* **errorreporting:** Remove dependency on AppEngine's proto definitions. This also removes the source_references field. ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) +* **errorreporting:** Update bazel builds for ER client libraries. ([0b4370a](https://www.github.com/googleapis/google-cloud-go/commit/0b4370a0d397913d932dbbdc2046a958dc3b836a)) +* **internal/godocfx:** use exact list of top-level decls ([#3665](https://www.github.com/googleapis/google-cloud-go/issues/3665)) ([3cd2961](https://www.github.com/googleapis/google-cloud-go/commit/3cd2961bd7b9c29d82a21ba8850eff00c7c332fd)) +* **kms:** do not retry on 13 INTERNAL ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) +* **orgpolicy:** Fix constraint resource pattern annotation ([f66114b](https://www.github.com/googleapis/google-cloud-go/commit/f66114bc7233ad06e18f38dd39497a74d85fdbd8)) +* **orgpolicy:** Fix constraint resource pattern annotation ([0b4370a](https://www.github.com/googleapis/google-cloud-go/commit/0b4370a0d397913d932dbbdc2046a958dc3b836a)) +* **profiler:** make sure retries use the most up-to-date copy of the trailer ([#3660](https://www.github.com/googleapis/google-cloud-go/issues/3660)) ([3ba9ebc](https://www.github.com/googleapis/google-cloud-go/commit/3ba9ebcee2b8b43cdf2c8f8a3d810516a604b363)) +* **vision:** sync vision v1 protos to get extra FaceAnnotation Landmark Types ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) + +## [0.76.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.75.0...v0.76.0) (2021-02-02) + + +### Features + +* **accessapproval:** Migrate the Bazel rules for the libraries published to google-cloud-ruby to use the gapic-generator-ruby instead of the monolith generator. ([ac22beb](https://www.github.com/googleapis/google-cloud-go/commit/ac22beb9b90771b24c8b35db7587ad3f5c0a970e)) +* **all:** auto-regenerate gapics ([#3526](https://www.github.com/googleapis/google-cloud-go/issues/3526)) ([ab2af0b](https://www.github.com/googleapis/google-cloud-go/commit/ab2af0b32630dd97f44800f4e273184f887375db)) +* **all:** auto-regenerate gapics ([#3539](https://www.github.com/googleapis/google-cloud-go/issues/3539)) ([84d4d8a](https://www.github.com/googleapis/google-cloud-go/commit/84d4d8ae2d3fbf34a4a312a0a2e4062d18caaa3d)) +* **all:** auto-regenerate gapics ([#3546](https://www.github.com/googleapis/google-cloud-go/issues/3546)) ([959fde5](https://www.github.com/googleapis/google-cloud-go/commit/959fde5ab12f7aee206dd46022e3cad1bc3470f7)) +* **all:** auto-regenerate gapics ([#3563](https://www.github.com/googleapis/google-cloud-go/issues/3563)) ([102112a](https://www.github.com/googleapis/google-cloud-go/commit/102112a4e9285a16645aabc89789f613d4f47c9e)) +* **all:** auto-regenerate gapics ([#3576](https://www.github.com/googleapis/google-cloud-go/issues/3576)) ([ac22beb](https://www.github.com/googleapis/google-cloud-go/commit/ac22beb9b90771b24c8b35db7587ad3f5c0a970e)) +* **all:** auto-regenerate gapics ([#3580](https://www.github.com/googleapis/google-cloud-go/issues/3580)) ([9974a80](https://www.github.com/googleapis/google-cloud-go/commit/9974a8017b5de8129a586f2404a23396caea0ee1)) +* **all:** auto-regenerate gapics ([#3587](https://www.github.com/googleapis/google-cloud-go/issues/3587)) ([3859a6f](https://www.github.com/googleapis/google-cloud-go/commit/3859a6ffc447e9c0b4ef231e2788fbbcfe48a94f)) +* **all:** auto-regenerate gapics ([#3598](https://www.github.com/googleapis/google-cloud-go/issues/3598)) ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **appengine:** start generating apiv1 ([#3561](https://www.github.com/googleapis/google-cloud-go/issues/3561)) ([2b6a3b4](https://www.github.com/googleapis/google-cloud-go/commit/2b6a3b4609e389da418a83eb60a8ae3710d646d7)) +* **assuredworkloads:** updated google.cloud.assuredworkloads.v1beta1.AssuredWorkloadsService service. Clients can now create workloads with US_REGIONAL_ACCESS compliance regime ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **binaryauthorization:** start generating apiv1beta1 ([#3562](https://www.github.com/googleapis/google-cloud-go/issues/3562)) ([56e18a6](https://www.github.com/googleapis/google-cloud-go/commit/56e18a64836ab9482528b212eb139f649f7a35c3)) +* **channel:** Add Pub/Sub endpoints for Cloud Channel API. ([9070c86](https://www.github.com/googleapis/google-cloud-go/commit/9070c86e2c69f9405d42fc0e6fe7afd4a256d8b8)) +* **cloudtasks:** introducing field: ListQueuesRequest.read_mask, GetQueueRequest.read_mask, Queue.task_ttl, Queue.tombstone_ttl, Queue.stats, Task.pull_message and introducing messages: QueueStats PullMessage docs: updates to max burst size description ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **cloudtasks:** introducing fields: ListQueuesRequest.read_mask, GetQueueRequest.read_mask, Queue.task_ttl, Queue.tombstone_ttl, Queue.stats and introducing messages: QueueStats docs: updates to AppEngineHttpRequest description ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **datalabeling:** start generating apiv1beta1 ([#3582](https://www.github.com/googleapis/google-cloud-go/issues/3582)) ([d8a7fee](https://www.github.com/googleapis/google-cloud-go/commit/d8a7feef51d3344fa7e258aba1d9fbdab56dadcf)) +* **dataqna:** start generating apiv1alpha ([#3586](https://www.github.com/googleapis/google-cloud-go/issues/3586)) ([24c5b8f](https://www.github.com/googleapis/google-cloud-go/commit/24c5b8f4f45f8cd8b3001b1ca5a8d80e9f3b39d5)) +* **dialogflow/cx:** Add new Experiment service docs: minor doc update on redact field in intent.proto and page.proto ([0959f27](https://www.github.com/googleapis/google-cloud-go/commit/0959f27e85efe94d39437ceef0ff62ddceb8e7a7)) +* **dialogflow/cx:** added support for test cases and agent validation ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **dialogflow/cx:** added support for test cases and agent validation ([3859a6f](https://www.github.com/googleapis/google-cloud-go/commit/3859a6ffc447e9c0b4ef231e2788fbbcfe48a94f)) +* **dialogflow:** add C++ targets for DialogFlow ([959fde5](https://www.github.com/googleapis/google-cloud-go/commit/959fde5ab12f7aee206dd46022e3cad1bc3470f7)) +* **documentai:** start generating apiv1beta3 ([#3595](https://www.github.com/googleapis/google-cloud-go/issues/3595)) ([5ae21fa](https://www.github.com/googleapis/google-cloud-go/commit/5ae21fa1cfb8b8dacbcd0fc43eee430f7db63102)) +* **domains:** start generating apiv1beta1 ([#3632](https://www.github.com/googleapis/google-cloud-go/issues/3632)) ([b8ada6f](https://www.github.com/googleapis/google-cloud-go/commit/b8ada6f197e680d0bb26aa031e6431bc099a3149)) +* **godocfx:** include alt documentation link ([#3530](https://www.github.com/googleapis/google-cloud-go/issues/3530)) ([806cdd5](https://www.github.com/googleapis/google-cloud-go/commit/806cdd56fb6fdddd7a6c1354e55e0d1259bd6c8b)) +* **internal/gapicgen:** change commit formatting to match standard ([#3500](https://www.github.com/googleapis/google-cloud-go/issues/3500)) ([d1e3d46](https://www.github.com/googleapis/google-cloud-go/commit/d1e3d46c47c425581e2b149c07f8e27ffc373c7e)) +* **internal/godocfx:** xref function declarations ([#3615](https://www.github.com/googleapis/google-cloud-go/issues/3615)) ([2bdbb87](https://www.github.com/googleapis/google-cloud-go/commit/2bdbb87a682d799cf5e262a61a3ef1faf41151af)) +* **mediatranslation:** start generating apiv1beta1 ([#3636](https://www.github.com/googleapis/google-cloud-go/issues/3636)) ([4129469](https://www.github.com/googleapis/google-cloud-go/commit/412946966cf7f53c51deff1b1cc1a12d62ed0279)) +* **memcache:** start generating apiv1 ([#3579](https://www.github.com/googleapis/google-cloud-go/issues/3579)) ([eabf7cf](https://www.github.com/googleapis/google-cloud-go/commit/eabf7cfde7b3a3cc1b35c320ba52e07be9926359)) +* **networkconnectivity:** initial generation of apiv1alpha1 ([#3567](https://www.github.com/googleapis/google-cloud-go/issues/3567)) ([adf489a](https://www.github.com/googleapis/google-cloud-go/commit/adf489a536292e3196677621477eae0d52761e7f)) +* **orgpolicy:** start generating apiv2 ([#3652](https://www.github.com/googleapis/google-cloud-go/issues/3652)) ([c103847](https://www.github.com/googleapis/google-cloud-go/commit/c1038475779fda3589aa9659d4ad0b703036b531)) +* **osconfig/agentendpoint:** add ApplyConfigTask to AgentEndpoint API ([9070c86](https://www.github.com/googleapis/google-cloud-go/commit/9070c86e2c69f9405d42fc0e6fe7afd4a256d8b8)) +* **osconfig/agentendpoint:** add ApplyConfigTask to AgentEndpoint API ([9af529c](https://www.github.com/googleapis/google-cloud-go/commit/9af529c21e98b62c4617f7a7191c307659cf8bb8)) +* **recommender:** add bindings for folder/org type resources for protos in recommendations, insights and recommender_service to enable v1 api for folder/org ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **recommender:** auto generated cl for enabling v1beta1 folder/org APIs and integration test ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) +* **resourcemanager:** start generating apiv2 ([#3575](https://www.github.com/googleapis/google-cloud-go/issues/3575)) ([93d0ebc](https://www.github.com/googleapis/google-cloud-go/commit/93d0ebceb4270351518a13958005bb68f0cace60)) +* **secretmanager:** added expire_time and ttl fields to Secret ([9974a80](https://www.github.com/googleapis/google-cloud-go/commit/9974a8017b5de8129a586f2404a23396caea0ee1)) +* **secretmanager:** added expire_time and ttl fields to Secret ([ac22beb](https://www.github.com/googleapis/google-cloud-go/commit/ac22beb9b90771b24c8b35db7587ad3f5c0a970e)) +* **servicecontrol:** start generating apiv1 ([#3644](https://www.github.com/googleapis/google-cloud-go/issues/3644)) ([f84938b](https://www.github.com/googleapis/google-cloud-go/commit/f84938bb4042a5629fd66bda42de028fd833648a)) +* **servicemanagement:** start generating apiv1 ([#3614](https://www.github.com/googleapis/google-cloud-go/issues/3614)) ([b96134f](https://www.github.com/googleapis/google-cloud-go/commit/b96134fe91c182237359000cd544af5fec60d7db)) + + +### Bug Fixes + +* **datacatalog:** Update PHP package name casing to match the PHP namespace in the proto files ([c7ecf0f](https://www.github.com/googleapis/google-cloud-go/commit/c7ecf0f3f454606b124e52d20af2545b2c68646f)) +* **internal/godocfx:** add TOC element for module root package ([#3599](https://www.github.com/googleapis/google-cloud-go/issues/3599)) ([1d6eb23](https://www.github.com/googleapis/google-cloud-go/commit/1d6eb238206fcf8815d88981527ef176851afd7a)) +* **profiler:** Force gax to retry in case of certificate errors ([#3178](https://www.github.com/googleapis/google-cloud-go/issues/3178)) ([35dcd72](https://www.github.com/googleapis/google-cloud-go/commit/35dcd725dcd03266ed7439de40c277376b38cd71)) + +## [0.75.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.74.0...v0.75.0) (2021-01-11) + + +### Features + +* **all:** auto-regenerate gapics , refs [#3514](https://www.github.com/googleapis/google-cloud-go/issues/3514) [#3501](https://www.github.com/googleapis/google-cloud-go/issues/3501) [#3497](https://www.github.com/googleapis/google-cloud-go/issues/3497) [#3455](https://www.github.com/googleapis/google-cloud-go/issues/3455) [#3448](https://www.github.com/googleapis/google-cloud-go/issues/3448) +* **channel:** start generating apiv1 ([#3517](https://www.github.com/googleapis/google-cloud-go/issues/3517)) ([2cf3b3c](https://www.github.com/googleapis/google-cloud-go/commit/2cf3b3cf7d99f2efd6868a710fad9e935fc87965)) + + +### Bug Fixes + +* **internal/gapicgen:** don't regen files that have been deleted ([#3471](https://www.github.com/googleapis/google-cloud-go/issues/3471)) ([112ca94](https://www.github.com/googleapis/google-cloud-go/commit/112ca9416cc8a2502b32547dc8d789655452f84a)) + +## [0.74.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.73.0...v0.74.0) (2020-12-10) + + +### Features + +* **all:** auto-regenerate gapics , refs [#3440](https://www.github.com/googleapis/google-cloud-go/issues/3440) [#3436](https://www.github.com/googleapis/google-cloud-go/issues/3436) [#3394](https://www.github.com/googleapis/google-cloud-go/issues/3394) [#3391](https://www.github.com/googleapis/google-cloud-go/issues/3391) [#3374](https://www.github.com/googleapis/google-cloud-go/issues/3374) +* **internal/gapicgen:** support generating only gapics with genlocal ([#3383](https://www.github.com/googleapis/google-cloud-go/issues/3383)) ([eaa742a](https://www.github.com/googleapis/google-cloud-go/commit/eaa742a248dc7d93c019863248f28e37f88aae84)) +* **servicedirectory:** start generating apiv1 ([#3382](https://www.github.com/googleapis/google-cloud-go/issues/3382)) ([2774925](https://www.github.com/googleapis/google-cloud-go/commit/2774925925909071ebc585cf7400373334c156ba)) + + +### Bug Fixes + +* **internal/gapicgen:** don't create genproto pr as draft ([#3379](https://www.github.com/googleapis/google-cloud-go/issues/3379)) ([517ab0f](https://www.github.com/googleapis/google-cloud-go/commit/517ab0f25e544498c5374b256354bc41ba936ad5)) + +## [0.73.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.72.0...v0.73.0) (2020-12-04) + + +### Features + +* **all:** auto-regenerate gapics , refs [#3335](https://www.github.com/googleapis/google-cloud-go/issues/3335) [#3294](https://www.github.com/googleapis/google-cloud-go/issues/3294) [#3250](https://www.github.com/googleapis/google-cloud-go/issues/3250) [#3229](https://www.github.com/googleapis/google-cloud-go/issues/3229) [#3211](https://www.github.com/googleapis/google-cloud-go/issues/3211) [#3217](https://www.github.com/googleapis/google-cloud-go/issues/3217) [#3212](https://www.github.com/googleapis/google-cloud-go/issues/3212) [#3209](https://www.github.com/googleapis/google-cloud-go/issues/3209) [#3206](https://www.github.com/googleapis/google-cloud-go/issues/3206) [#3199](https://www.github.com/googleapis/google-cloud-go/issues/3199) +* **artifactregistry:** start generating apiv1beta2 ([#3352](https://www.github.com/googleapis/google-cloud-go/issues/3352)) ([2e6f20b](https://www.github.com/googleapis/google-cloud-go/commit/2e6f20b0ab438b0b366a1a3802fc64d1a0e66fff)) +* **internal:** copy pubsub Message and PublishResult to internal/pubsub ([#3351](https://www.github.com/googleapis/google-cloud-go/issues/3351)) ([82521ee](https://www.github.com/googleapis/google-cloud-go/commit/82521ee5038735c1663525658d27e4df00ec90be)) +* **internal/gapicgen:** support adding context to regen ([#3174](https://www.github.com/googleapis/google-cloud-go/issues/3174)) ([941ab02](https://www.github.com/googleapis/google-cloud-go/commit/941ab029ba6f7f33e8b2e31e3818aeb68312a999)) +* **internal/kokoro:** add ability to regen all DocFX YAML ([#3191](https://www.github.com/googleapis/google-cloud-go/issues/3191)) ([e12046b](https://www.github.com/googleapis/google-cloud-go/commit/e12046bc4431d33aee72c324e6eb5cc907a4214a)) + + +### Bug Fixes + +* **internal/godocfx:** filter out test packages from other modules ([#3197](https://www.github.com/googleapis/google-cloud-go/issues/3197)) ([1d397aa](https://www.github.com/googleapis/google-cloud-go/commit/1d397aa8b41f8f980cba1d3dcc50f11e4d4f4ca0)) + +## [0.72.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.71.0...v0.72.0) (2020-11-10) + + +### Features + +* **all:** auto-regenerate gapics , refs [#3177](https://www.github.com/googleapis/google-cloud-go/issues/3177) [#3164](https://www.github.com/googleapis/google-cloud-go/issues/3164) [#3149](https://www.github.com/googleapis/google-cloud-go/issues/3149) [#3142](https://www.github.com/googleapis/google-cloud-go/issues/3142) [#3136](https://www.github.com/googleapis/google-cloud-go/issues/3136) [#3130](https://www.github.com/googleapis/google-cloud-go/issues/3130) [#3121](https://www.github.com/googleapis/google-cloud-go/issues/3121) [#3119](https://www.github.com/googleapis/google-cloud-go/issues/3119) + + +### Bug Fixes + +* **all:** Update hand-written clients to not use WithEndpoint override ([#3111](https://www.github.com/googleapis/google-cloud-go/issues/3111)) ([f0cfd05](https://www.github.com/googleapis/google-cloud-go/commit/f0cfd0532f5204ff16f7bae406efa72603d16f44)) +* **internal/godocfx:** rename README files to pkg-readme ([#3185](https://www.github.com/googleapis/google-cloud-go/issues/3185)) ([d3a8571](https://www.github.com/googleapis/google-cloud-go/commit/d3a85719be411b692aede3331abb29b5a7b3da9a)) + + +## [0.71.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.70.0...v0.71.0) (2020-10-30) + + +### Features + +* **all:** auto-regenerate gapics , refs [#3115](https://www.github.com/googleapis/google-cloud-go/issues/3115) [#3106](https://www.github.com/googleapis/google-cloud-go/issues/3106) [#3102](https://www.github.com/googleapis/google-cloud-go/issues/3102) [#3083](https://www.github.com/googleapis/google-cloud-go/issues/3083) [#3073](https://www.github.com/googleapis/google-cloud-go/issues/3073) [#3057](https://www.github.com/googleapis/google-cloud-go/issues/3057) [#3044](https://www.github.com/googleapis/google-cloud-go/issues/3044) +* **billing/budgets:** start generating apiv1 ([#3099](https://www.github.com/googleapis/google-cloud-go/issues/3099)) ([e760c85](https://www.github.com/googleapis/google-cloud-go/commit/e760c859de88a6e79b6dffc653dbf75f1630d8e3)) +* **internal:** auto-run godocfx on new mods ([#3069](https://www.github.com/googleapis/google-cloud-go/issues/3069)) ([49f497e](https://www.github.com/googleapis/google-cloud-go/commit/49f497eab80ce34dfb4ca41f033a5c0429ff5e42)) +* **pubsublite:** Added Pub/Sub Lite clients and routing headers ([#3105](https://www.github.com/googleapis/google-cloud-go/issues/3105)) ([98668fa](https://www.github.com/googleapis/google-cloud-go/commit/98668fa5457d26ed34debee708614f027020e5bc)) +* **pubsublite:** Message type and message routers ([#3077](https://www.github.com/googleapis/google-cloud-go/issues/3077)) ([179fc55](https://www.github.com/googleapis/google-cloud-go/commit/179fc550b545a5344358a243da7007ffaa7b5171)) +* **pubsublite:** Pub/Sub Lite admin client ([#3036](https://www.github.com/googleapis/google-cloud-go/issues/3036)) ([749473e](https://www.github.com/googleapis/google-cloud-go/commit/749473ead30bf1872634821d3238d1299b99acc6)) +* **pubsublite:** Publish settings and errors ([#3075](https://www.github.com/googleapis/google-cloud-go/issues/3075)) ([9eb9fcb](https://www.github.com/googleapis/google-cloud-go/commit/9eb9fcb79f17ad7c08c77c455ba3e8d89e3bdbf2)) +* **pubsublite:** Retryable stream wrapper ([#3068](https://www.github.com/googleapis/google-cloud-go/issues/3068)) ([97cfd45](https://www.github.com/googleapis/google-cloud-go/commit/97cfd4587f2f51996bd685ff486308b70eb51900)) + + +### Bug Fixes + +* **internal/kokoro:** remove unnecessary cd ([#3071](https://www.github.com/googleapis/google-cloud-go/issues/3071)) ([c1a4c3e](https://www.github.com/googleapis/google-cloud-go/commit/c1a4c3eaffcdc3cffe0e223fcfa1f60879cd23bb)) +* **pubsublite:** Disable integration tests for project id ([#3087](https://www.github.com/googleapis/google-cloud-go/issues/3087)) ([a0982f7](https://www.github.com/googleapis/google-cloud-go/commit/a0982f79d6461feabdf31363f29fed7dc5677fe7)) + +## [0.70.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.69.0...v0.70.0) (2020-10-19) + + +### Features + +* **all:** auto-regenerate gapics , refs [#3047](https://www.github.com/googleapis/google-cloud-go/issues/3047) [#3035](https://www.github.com/googleapis/google-cloud-go/issues/3035) [#3025](https://www.github.com/googleapis/google-cloud-go/issues/3025) +* **managedidentities:** start generating apiv1 ([#3032](https://www.github.com/googleapis/google-cloud-go/issues/3032)) ([10ccca2](https://www.github.com/googleapis/google-cloud-go/commit/10ccca238074d24fea580a4cd8e64478818b0b44)) +* **pubsublite:** Types for resource paths and topic/subscription configs ([#3026](https://www.github.com/googleapis/google-cloud-go/issues/3026)) ([6f7fa86](https://www.github.com/googleapis/google-cloud-go/commit/6f7fa86ed906258f98d996aab40184f3a46f9714)) + +## [0.69.1](https://www.github.com/googleapis/google-cloud-go/compare/v0.69.0...v0.69.1) (2020-10-14) + +This is an empty release that was created solely to aid in pubsublite's module +carve out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## [0.69.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.68.0...v0.69.0) (2020-10-14) + + +### Features + +* **accessapproval:** start generating apiv1 ([#3002](https://www.github.com/googleapis/google-cloud-go/issues/3002)) ([709d6e7](https://www.github.com/googleapis/google-cloud-go/commit/709d6e76393e6ac00ff488efd83bfe873173b045)) +* **all:** auto-regenerate gapics , refs [#3010](https://www.github.com/googleapis/google-cloud-go/issues/3010) [#3005](https://www.github.com/googleapis/google-cloud-go/issues/3005) [#2993](https://www.github.com/googleapis/google-cloud-go/issues/2993) [#2989](https://www.github.com/googleapis/google-cloud-go/issues/2989) [#2981](https://www.github.com/googleapis/google-cloud-go/issues/2981) [#2976](https://www.github.com/googleapis/google-cloud-go/issues/2976) [#2968](https://www.github.com/googleapis/google-cloud-go/issues/2968) [#2958](https://www.github.com/googleapis/google-cloud-go/issues/2958) +* **cmd/go-cloud-debug-agent:** mark as deprecated ([#2964](https://www.github.com/googleapis/google-cloud-go/issues/2964)) ([276ec88](https://www.github.com/googleapis/google-cloud-go/commit/276ec88b05852c33a3ba437e18d072f7ffd8fd33)) +* **godocfx:** add nesting to TOC ([#2972](https://www.github.com/googleapis/google-cloud-go/issues/2972)) ([3a49b2d](https://www.github.com/googleapis/google-cloud-go/commit/3a49b2d142a353f98429235c3f380431430b4dbf)) +* **internal/godocfx:** HTML-ify package summary ([#2986](https://www.github.com/googleapis/google-cloud-go/issues/2986)) ([9e64b01](https://www.github.com/googleapis/google-cloud-go/commit/9e64b018255bd8d9b31d60e8f396966251de946b)) +* **internal/kokoro:** make publish_docs VERSION optional ([#2979](https://www.github.com/googleapis/google-cloud-go/issues/2979)) ([76e35f6](https://www.github.com/googleapis/google-cloud-go/commit/76e35f689cb60bd5db8e14b8c8d367c5902bcb0e)) +* **websecurityscanner:** start generating apiv1 ([#3006](https://www.github.com/googleapis/google-cloud-go/issues/3006)) ([1d92e20](https://www.github.com/googleapis/google-cloud-go/commit/1d92e2062a13f62d7a96be53a7354c0cacca6a85)) + + +### Bug Fixes + +* **godocfx:** make extra files optional, filter out third_party ([#2985](https://www.github.com/googleapis/google-cloud-go/issues/2985)) ([f268921](https://www.github.com/googleapis/google-cloud-go/commit/f2689214a24b2e325d3e8f54441bb11fbef925f0)) + +## [0.68.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.67.0...v0.68.0) (2020-10-02) + + +### Features + +* **all:** auto-regenerate gapics , refs [#2952](https://www.github.com/googleapis/google-cloud-go/issues/2952) [#2944](https://www.github.com/googleapis/google-cloud-go/issues/2944) [#2935](https://www.github.com/googleapis/google-cloud-go/issues/2935) + +## [0.67.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.66.0...v0.67.0) (2020-09-29) + + +### Features + +* **all:** auto-regenerate gapics , refs [#2933](https://www.github.com/googleapis/google-cloud-go/issues/2933) [#2919](https://www.github.com/googleapis/google-cloud-go/issues/2919) [#2913](https://www.github.com/googleapis/google-cloud-go/issues/2913) [#2910](https://www.github.com/googleapis/google-cloud-go/issues/2910) [#2899](https://www.github.com/googleapis/google-cloud-go/issues/2899) [#2897](https://www.github.com/googleapis/google-cloud-go/issues/2897) [#2886](https://www.github.com/googleapis/google-cloud-go/issues/2886) [#2877](https://www.github.com/googleapis/google-cloud-go/issues/2877) [#2869](https://www.github.com/googleapis/google-cloud-go/issues/2869) [#2864](https://www.github.com/googleapis/google-cloud-go/issues/2864) +* **assuredworkloads:** start generating apiv1beta1 ([#2866](https://www.github.com/googleapis/google-cloud-go/issues/2866)) ([7598c4d](https://www.github.com/googleapis/google-cloud-go/commit/7598c4dd2462e8270a2c7b1f496af58ca81ff568)) +* **dialogflow/cx:** start generating apiv3beta1 ([#2875](https://www.github.com/googleapis/google-cloud-go/issues/2875)) ([37ca93a](https://www.github.com/googleapis/google-cloud-go/commit/37ca93ad69eda363d956f0174d444ed5914f5a72)) +* **docfx:** add support for examples ([#2884](https://www.github.com/googleapis/google-cloud-go/issues/2884)) ([0cc0de3](https://www.github.com/googleapis/google-cloud-go/commit/0cc0de300d58be6d3b7eeb2f1baebfa6df076830)) +* **godocfx:** include README in output ([#2927](https://www.github.com/googleapis/google-cloud-go/issues/2927)) ([f084690](https://www.github.com/googleapis/google-cloud-go/commit/f084690a2ea08ce73bafaaced95ad271fd01e11e)) +* **talent:** start generating apiv4 ([#2871](https://www.github.com/googleapis/google-cloud-go/issues/2871)) ([5c98071](https://www.github.com/googleapis/google-cloud-go/commit/5c98071b03822c58862d1fa5442ff36d627f1a61)) + + +### Bug Fixes + +* **godocfx:** filter out other modules, sort pkgs ([#2894](https://www.github.com/googleapis/google-cloud-go/issues/2894)) ([868db45](https://www.github.com/googleapis/google-cloud-go/commit/868db45e2e6f4e9ad48432be86c849f335e1083d)) +* **godocfx:** shorten function names ([#2880](https://www.github.com/googleapis/google-cloud-go/issues/2880)) ([48a0217](https://www.github.com/googleapis/google-cloud-go/commit/48a0217930750c1f4327f2622b0f2a3ec8afc0b7)) +* **translate:** properly name examples ([#2892](https://www.github.com/googleapis/google-cloud-go/issues/2892)) ([c19e141](https://www.github.com/googleapis/google-cloud-go/commit/c19e1415e6fa76b7ea66a7fc67ad3ba22670a2ba)), refs [#2883](https://www.github.com/googleapis/google-cloud-go/issues/2883) + +## [0.66.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.65.0...v0.66.0) (2020-09-15) + + +### Features + +* **all:** auto-regenerate gapics , refs [#2849](https://www.github.com/googleapis/google-cloud-go/issues/2849) [#2843](https://www.github.com/googleapis/google-cloud-go/issues/2843) [#2841](https://www.github.com/googleapis/google-cloud-go/issues/2841) [#2819](https://www.github.com/googleapis/google-cloud-go/issues/2819) [#2816](https://www.github.com/googleapis/google-cloud-go/issues/2816) [#2809](https://www.github.com/googleapis/google-cloud-go/issues/2809) [#2801](https://www.github.com/googleapis/google-cloud-go/issues/2801) [#2795](https://www.github.com/googleapis/google-cloud-go/issues/2795) [#2791](https://www.github.com/googleapis/google-cloud-go/issues/2791) [#2788](https://www.github.com/googleapis/google-cloud-go/issues/2788) [#2781](https://www.github.com/googleapis/google-cloud-go/issues/2781) +* **analytics/data:** start generating apiv1alpha ([#2796](https://www.github.com/googleapis/google-cloud-go/issues/2796)) ([e93132c](https://www.github.com/googleapis/google-cloud-go/commit/e93132c77725de3c80c34d566df269eabfcfde93)) +* **area120/tables:** start generating apiv1alpha1 ([#2807](https://www.github.com/googleapis/google-cloud-go/issues/2807)) ([9e5a4d0](https://www.github.com/googleapis/google-cloud-go/commit/9e5a4d0dee0d83be0c020797a2f579d9e42ef521)) +* **cloudbuild:** Start generating apiv1/v3 ([#2830](https://www.github.com/googleapis/google-cloud-go/issues/2830)) ([358a536](https://www.github.com/googleapis/google-cloud-go/commit/358a5368da64cf4868551652e852ceb453504f64)) +* **godocfx:** create Go DocFX YAML generator ([#2854](https://www.github.com/googleapis/google-cloud-go/issues/2854)) ([37c70ac](https://www.github.com/googleapis/google-cloud-go/commit/37c70acd91768567106ff3b2b130835998d974c5)) +* **security/privateca:** start generating apiv1beta1 ([#2806](https://www.github.com/googleapis/google-cloud-go/issues/2806)) ([f985141](https://www.github.com/googleapis/google-cloud-go/commit/f9851412183989dc69733a7e61ad39a9378cd893)) +* **video/transcoder:** start generating apiv1beta1 ([#2797](https://www.github.com/googleapis/google-cloud-go/issues/2797)) ([390dda8](https://www.github.com/googleapis/google-cloud-go/commit/390dda8ff2c526e325e434ad0aec778b7aa97ea4)) +* **workflows:** start generating apiv1beta ([#2799](https://www.github.com/googleapis/google-cloud-go/issues/2799)) ([0e39665](https://www.github.com/googleapis/google-cloud-go/commit/0e39665ccb788caec800e2887d433ca6e0cf9901)) +* **workflows/executions:** start generating apiv1beta ([#2800](https://www.github.com/googleapis/google-cloud-go/issues/2800)) ([7eaa0d1](https://www.github.com/googleapis/google-cloud-go/commit/7eaa0d184c6a2141d8bf4514b3fd20715b50a580)) + + +### Bug Fixes + +* **internal/kokoro:** install the right version of docuploader ([#2861](https://www.github.com/googleapis/google-cloud-go/issues/2861)) ([d8489c1](https://www.github.com/googleapis/google-cloud-go/commit/d8489c141b8b02e83d6426f4baebd3658ae11639)) +* **internal/kokoro:** remove extra dash in doc tarball ([#2862](https://www.github.com/googleapis/google-cloud-go/issues/2862)) ([690ddcc](https://www.github.com/googleapis/google-cloud-go/commit/690ddccc5202b5a70f1afa5c518dca37b6a0861c)) +* **profiler:** do not collect disabled profile types ([#2836](https://www.github.com/googleapis/google-cloud-go/issues/2836)) ([faeb498](https://www.github.com/googleapis/google-cloud-go/commit/faeb4985bf6afdcddba4553efa874642bf7f08ed)), refs [#2835](https://www.github.com/googleapis/google-cloud-go/issues/2835) + + +### Reverts + +* **cloudbuild): "feat(cloudbuild:** Start generating apiv1/v3" ([#2840](https://www.github.com/googleapis/google-cloud-go/issues/2840)) ([3aaf755](https://www.github.com/googleapis/google-cloud-go/commit/3aaf755476dfea1700986fc086f53fc1ab756557)) + +## [0.65.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.64.0...v0.65.0) (2020-08-27) + + +### Announcements + +The following changes will be included in an upcoming release and are not +included in this one. + +#### Default Deadlines + +By default, non-streaming methods, like Create or Get methods, will have a +default deadline applied to the context provided at call time, unless a context +deadline is already set. Streaming methods have no default deadline and will run +indefinitely, unless the context provided at call time contains a deadline. + +To opt-out of this behavior, set the environment variable +`GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE` to `true` prior to +initializing a client. This opt-out mechanism will be removed in a later +release, with a notice similar to this one ahead of its removal. + + +### Features + +* **all:** auto-regenerate gapics , refs [#2774](https://www.github.com/googleapis/google-cloud-go/issues/2774) [#2764](https://www.github.com/googleapis/google-cloud-go/issues/2764) + + +### Bug Fixes + +* **all:** correct minor typos ([#2756](https://www.github.com/googleapis/google-cloud-go/issues/2756)) ([03d78b5](https://www.github.com/googleapis/google-cloud-go/commit/03d78b5627819cb64d1f3866f90043f709e825e1)) +* **compute/metadata:** remove leading slash for Get suffix ([#2760](https://www.github.com/googleapis/google-cloud-go/issues/2760)) ([f0d605c](https://www.github.com/googleapis/google-cloud-go/commit/f0d605ccf32391a9da056a2c551158bd076c128d)) + +## [0.64.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.63.0...v0.64.0) (2020-08-18) + + +### Features + +* **all:** auto-regenerate gapics , refs [#2734](https://www.github.com/googleapis/google-cloud-go/issues/2734) [#2731](https://www.github.com/googleapis/google-cloud-go/issues/2731) [#2730](https://www.github.com/googleapis/google-cloud-go/issues/2730) [#2725](https://www.github.com/googleapis/google-cloud-go/issues/2725) [#2722](https://www.github.com/googleapis/google-cloud-go/issues/2722) [#2706](https://www.github.com/googleapis/google-cloud-go/issues/2706) +* **pubsublite:** start generating v1 ([#2700](https://www.github.com/googleapis/google-cloud-go/issues/2700)) ([d2e777f](https://www.github.com/googleapis/google-cloud-go/commit/d2e777f56e08146646b3ffb7a78856795094ab4e)) + +## [0.63.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.62.0...v0.63.0) (2020-08-05) + + +### Features + +* **all:** auto-regenerate gapics ([#2682](https://www.github.com/googleapis/google-cloud-go/issues/2682)) ([63bfd63](https://www.github.com/googleapis/google-cloud-go/commit/63bfd638da169e0f1f4fa4a5125da2955022dc04)) +* **analytics/admin:** start generating apiv1alpha ([#2670](https://www.github.com/googleapis/google-cloud-go/issues/2670)) ([268199e](https://www.github.com/googleapis/google-cloud-go/commit/268199e5350a64a83ecf198e0e0fa4863f00fa6c)) +* **functions/metadata:** Special-case marshaling ([#2669](https://www.github.com/googleapis/google-cloud-go/issues/2669)) ([d8d7fc6](https://www.github.com/googleapis/google-cloud-go/commit/d8d7fc66cbc42f79bec25fb0daaf53d926e3645b)) +* **gaming:** start generate apiv1 ([#2681](https://www.github.com/googleapis/google-cloud-go/issues/2681)) ([1adfd0a](https://www.github.com/googleapis/google-cloud-go/commit/1adfd0aed6b2c0e1dd0c575a5ec0f49388fa5601)) +* **internal/kokoro:** add script to test compatibility with samples ([#2637](https://www.github.com/googleapis/google-cloud-go/issues/2637)) ([f2aa76a](https://www.github.com/googleapis/google-cloud-go/commit/f2aa76a0058e86c1c33bb634d2c084b58f77ab32)) + +## v0.62.0 + +### Announcements + +- There was a breaking change to `cloud.google.com/go/dataproc/apiv1` that was + merged in [this PR](https://github.com/googleapis/google-cloud-go/pull/2606). + This fixed a broken API response for `DiagnoseCluster`. When polling on the + Long Running Operation(LRO), the API now returns + `(*dataprocpb.DiagnoseClusterResults, error)` whereas it only returned an + `error` before. + +### Changes + +- all: + - Updated all direct dependencies. + - Updated contributing guidelines to suggest allowing edits from maintainers. +- billing/budgets: + - Start generating client for apiv1beta1. +- functions: + - Start generating client for apiv1. +- notebooks: + - Start generating client apiv1beta1. +- profiler: + - update proftest to support parsing floating-point backoff durations. + - Fix the regexp used to parse backoff duration. +- Various updates to autogenerated clients. + +## v0.61.0 + +### Changes + +- all: + - Update all direct dependencies. +- dashboard: + - Start generating client for apiv1. +- policytroubleshooter: + - Start generating client for apiv1. +- profiler: + - Disable OpenCensus Telemetry for requests made by the profiler package by default. You can re-enable it using `profiler.Config.EnableOCTelemetry`. +- Various updates to autogenerated clients. + +## v0.60.0 + +### Changes + +- all: + - Refactored examples to reduce module dependencies. + - Update sub-modules to use cloud.google.com/go v0.59.0. +- internal: + - Start generating client for gaming apiv1beta. +- Various updates to autogenerated clients. + +## v0.59.0 + +### Announcements + +goolgeapis/google-cloud-go has moved its source of truth to GitHub and is no longer a mirror. This means that our +contributing process has changed a bit. We will now be conducting all code reviews on GitHub which means we now accept +pull requests! If you have a version of the codebase previously checked out you may wish to update your git remote to +point to GitHub. + +### Changes + +- all: + - Remove dependency on honnef.co/go/tools. + - Update our contributing instructions now that we use GitHub for reviews. + - Remove some un-inclusive terminology. +- compute/metadata: + - Pass cancelable context to DNS lookup. +- .github: + - Update templates issue/PR templates. +- internal: + - Bump several clients to GA. + - Fix GoDoc badge source. + - Several automation changes related to the move to GitHub. + - Start generating a client for asset v1p5beta1. +- Various updates to autogenerated clients. + +## v0.58.0 + +### Deprecation notice + +- `cloud.google.com/go/monitoring/apiv3` has been deprecated due to breaking + changes in the API. Please migrate to `cloud.google.com/go/monitoring/apiv3/v2`. + +### Changes + +- all: + - The remaining uses of gtransport.Dial have been removed. + - The `genproto` dependency has been updated to a version that makes use of + new `protoreflect` library. For more information on these protobuf changes + please see the following post from the official Go blog: + https://blog.golang.org/protobuf-apiv2. +- internal: + - Started generation of datastore admin v1 client. + - Updated protofuf version used for generation to 3.12.X. + - Update the release levels for several APIs. + - Generate clients with protoc-gen-go@v1.4.1. +- monitoring: + - Re-enable generation of monitoring/apiv3 under v2 directory (see deprecation + notice above). +- profiler: + - Fixed flakiness in tests. +- Various updates to autogenerated clients. + +## v0.57.0 + +- all: + - Update module dependency `google.golang.org/api` to `v0.21.0`. +- errorreporting: + - Add exported SetGoogleClientInfo wrappers to manual file. +- expr/v1alpha1: + - Deprecate client. This client will be removed in a future release. +- internal: + - Fix possible data race in TestTracer. + - Pin versions of tools used for generation. + - Correct the release levels for BigQuery APIs. + - Start generation osconfig v1. +- longrunning: + - Add exported SetGoogleClientInfo wrappers to manual file. +- monitoring: + - Stop generation of monitoring/apiv3 because of incoming breaking change. +- trace: + - Add exported SetGoogleClientInfo wrappers to manual file. +- Various updates to autogenerated clients. + +## v0.56.0 + +- secretmanager: + - add IAM helper +- profiler: + - try all us-west1 zones for integration tests +- internal: + - add config to generate webrisk v1 + - add repo and commit to buildcop invocation + - add recaptchaenterprise v1 generation config + - update microgenerator to v0.12.5 + - add datacatalog client + - start generating security center settings v1beta + - start generating osconfig agentendpoint v1 + - setup generation for bigquery/connection/v1beta1 +- all: + - increase continous testing timeout to 45m + - various updates to autogenerated clients. + +## v0.55.0 + +- Various updates to autogenerated clients. + +## v0.54.0 + +- all: + - remove unused golang.org/x/exp from mod file + - update godoc.org links to pkg.go.dev +- compute/metadata: + - use defaultClient when http.Client is nil + - remove subscribeClient +- iam: + - add support for v3 policy and IAM conditions +- Various updates to autogenerated clients. + +## v0.53.0 + +- all: most clients now use transport/grpc.DialPool rather than Dial (see #1777 for outliers). + - Connection pooling now does not use the deprecated (and soon to be removed) gRPC load balancer API. +- profiler: remove symbolization (drops support for go1.10) +- Various updates to autogenerated clients. + +## v0.52.0 + +- internal/gapicgen: multiple improvements related to library generation. +- compute/metadata: unset ResponseHeaderTimeout in defaultClient +- docs: fix link to KMS in README.md +- Various updates to autogenerated clients. + +## v0.51.0 + +- secretmanager: + - add IAM helper for generic resource IAM handle +- cloudbuild: + - migrate to microgen in a major version +- Various updates to autogenerated clients. + +## v0.50.0 + +- profiler: + - Support disabling CPU profile collection. + - Log when a profile creation attempt begins. +- compute/metadata: + - Fix panic on malformed URLs. + - InstanceName returns actual instance name. +- Various updates to autogenerated clients. + +## v0.49.0 + +- functions/metadata: + - Handle string resources in JSON unmarshaller. +- Various updates to autogenerated clients. + +## v0.48.0 + +- Various updates to autogenerated clients + +## v0.47.0 + +This release drops support for Go 1.9 and Go 1.10: we continue to officially +support Go 1.11, Go 1.12, and Go 1.13. + +- Various updates to autogenerated clients. +- Add cloudbuild/apiv1 client. + +## v0.46.3 + +This is an empty release that was created solely to aid in storage's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.46.2 + +This is an empty release that was created solely to aid in spanner's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.46.1 + +This is an empty release that was created solely to aid in firestore's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.46.0 + +- spanner: + - Retry "Session not found" for read-only transactions. + - Retry aborted PDMLs. +- spanner/spannertest: + - Fix a bug that was causing 0X-prefixed number to be parsed incorrectly. +- storage: + - Add HMACKeyOptions. + - Remove *REGIONAL from StorageClass documentation. Using MULTI_REGIONAL, + DURABLE_REDUCED_AVAILABILITY, and REGIONAL are no longer best practice + StorageClasses but they are still acceptable values. +- trace: + - Remove cloud.google.com/go/trace. Package cloud.google.com/go/trace has been + marked OBSOLETE for several years: it is now no longer provided. If you + relied on this package, please vendor it or switch to using + https://cloud.google.com/trace/docs/setup/go (which obsoleted it). + +## v0.45.1 + +This is an empty release that was created solely to aid in pubsub's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.45.0 + +- compute/metadata: + - Add Email method. +- storage: + - Fix duplicated retry logic. + - Add ReaderObjectAttrs.StartOffset. + - Support reading last N bytes of a file when a negative range is given, such + as `obj.NewRangeReader(ctx, -10, -1)`. + - Add HMACKey listing functionality. +- spanner/spannertest: + - Support primary keys with no columns. + - Fix MinInt64 parsing. + - Implement deletion of key ranges. + - Handle reads during a read-write transaction. + - Handle returning DATE values. +- pubsub: + - Fix Ack/Modack request size calculation. +- logging: + - Add auto-detection of monitored resources on GAE Standard. + +## v0.44.3 + +This is an empty release that was created solely to aid in bigtable's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.44.2 + +This is an empty release that was created solely to aid in bigquery's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.44.1 + +This is an empty release that was created solely to aid in datastore's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.44.0 + +- datastore: + - Interface elements whose underlying types are supported, are now supported. + - Reduce time to initial retry from 1s to 100ms. +- firestore: + - Add Increment transformation. +- storage: + - Allow emulator with STORAGE_EMULATOR_HOST. + - Add methods for HMAC key management. +- pubsub: + - Add PublishCount and PublishLatency measurements. + - Add DefaultPublishViews and DefaultSubscribeViews for convenience of + importing all views. + - Add add Subscription.PushConfig.AuthenticationMethod. +- spanner: + - Allow emulator usage with SPANNER_EMULATOR_HOST. + - Add cloud.google.com/go/spanner/spannertest, a spanner emulator. + - Add cloud.google.com/go/spanner/spansql which contains types and a parser + for the Cloud Spanner SQL dialect. +- asset: + - Add apiv1p2beta1 client. + +## v0.43.0 + +This is an empty release that was created solely to aid in logging's module +carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. + +## v0.42.0 + +- bigtable: + - Add an admin method to update an instance and clusters. + - Fix bttest regex matching behavior for alternations (things like `|a`). + - Expose BlockAllFilter filter. +- bigquery: + - Add Routines API support. +- storage: + - Add read-only Bucket.LocationType. +- logging: + - Add TraceSampled to Entry. + - Fix to properly extract {Trace, Span}Id from X-Cloud-Trace-Context. +- pubsub: + - Add Cloud Key Management to TopicConfig. + - Change ExpirationPolicy to optional.Duration. +- automl: + - Add apiv1beta1 client. +- iam: + - Fix compilation problem with iam/credentials/apiv1. + +## v0.41.0 + +- bigtable: + - Check results from PredicateFilter in bttest, which fixes certain false matches. +- profiler: + - debugLog checks user defined logging options before logging. +- spanner: + - PartitionedUpdates respect query parameters. + - StartInstance allows specifying cloud API access scopes. +- bigquery: + - Use empty slice instead of nil for ValueSaver, fixing an issue with zero-length, repeated, nested fields causing panics. +- firestore: + - Return same number of snapshots as doc refs (in the form of duplicate records) during GetAll. +- replay: + - Change references to IPv4 addresses to localhost, making replay compatible with IPv6. + +## v0.40.0 + +- all: + - Update to protobuf-golang v1.3.1. +- datastore: + - Attempt to decode GAE-encoded keys if initial decoding attempt fails. + - Support integer time conversion. +- pubsub: + - Add PublishSettings.BundlerByteLimit. If users receive pubsub.ErrOverflow, + this value should be adjusted higher. + - Use IPv6 compatible target in testutil. +- bigtable: + - Fix Latin-1 regexp filters in bttest, allowing \C. + - Expose PassAllFilter. +- profiler: + - Add log messages for slow path in start. + - Fix start to allow retry until success. +- firestore: + - Add admin client. +- containeranalysis: + - Add apiv1 client. +- grafeas: + - Add apiv1 client. + +## 0.39.0 + +- bigtable: + - Implement DeleteInstance in bttest. + - Return an error on invalid ReadRowsRequest.RowRange key ranges in bttest. +- bigquery: + - Move RequirePartitionFilter outside of TimePartioning. + - Expose models API. +- firestore: + - Allow array values in create and update calls. + - Add CollectionGroup method. +- pubsub: + - Add ExpirationPolicy to Subscription. +- storage: + - Add V4 signing. +- rpcreplay: + - Match streams by first sent request. This further improves rpcreplay's + ability to distinguish streams. +- httpreplay: + - Set up Man-In-The-Middle config only once. This should improve proxy + creation when multiple proxies are used in a single process. + - Remove error on empty Content-Type, allowing requests with no Content-Type + header but a non-empty body. +- all: + - Fix an edge case bug in auto-generated library pagination by properly + propagating pagetoken. + +## 0.38.0 + +This update includes a substantial reduction in our transitive dependency list +by way of updating to opencensus@v0.21.0. + +- spanner: + - Error implements GRPCStatus, allowing status.Convert. +- bigtable: + - Fix a bug in bttest that prevents single column queries returning results + that match other filters. + - Remove verbose retry logging. +- logging: + - Ensure RequestUrl has proper UTF-8, removing the need for users to wrap and + rune replace manually. +- recaptchaenterprise: + - Add v1beta1 client. +- phishingprotection: + - Add v1beta1 client. + +## 0.37.4 + +This patch releases re-builds the go.sum. This was not possible in the +previous release. + +- firestore: + - Add sentinel value DetectProjectID for auto-detecting project ID. + - Add OpenCensus tracing for public methods. + - Marked stable. All future changes come with a backwards compatibility + guarantee. + - Removed firestore/apiv1beta1. All users relying on this low-level library + should migrate to firestore/apiv1. Note that most users should use the + high-level firestore package instead. +- pubsub: + - Allow large messages in synchronous pull case. + - Cap bundler byte limit. This should prevent OOM conditions when there are + a very large number of message publishes occurring. +- storage: + - Add ETag to BucketAttrs and ObjectAttrs. +- datastore: + - Removed some non-sensical OpenCensus traces. +- webrisk: + - Add v1 client. +- asset: + - Add v1 client. +- cloudtasks: + - Add v2 client. + +## 0.37.3 + +This patch release removes github.com/golang/lint from the transitive +dependency list, resolving `go get -u` problems. + +Note: this release intentionally has a broken go.sum. Please use v0.37.4. + +## 0.37.2 + +This patch release is mostly intended to bring in v0.3.0 of +google.golang.org/api, which fixes a GCF deployment issue. + +Note: we had to-date accidentally marked Redis as stable. In this release, we've +fixed it by downgrading its documentation to alpha, as it is in other languages +and docs. + +- all: + - Document context in generated libraries. + +## 0.37.1 + +Small go.mod version bumps to bring in v0.2.0 of google.golang.org/api, which +introduces a new oauth2 url. + +## 0.37.0 + +- spanner: + - Add BatchDML method. + - Reduced initial time between retries. +- bigquery: + - Produce better error messages for InferSchema. + - Add logical type control for avro loads. + - Add support for the GEOGRAPHY type. +- datastore: + - Add sentinel value DetectProjectID for auto-detecting project ID. + - Allow flatten tag on struct pointers. + - Fixed a bug that caused queries to panic with invalid queries. Instead they + will now return an error. +- profiler: + - Add ability to override GCE zone and instance. +- pubsub: + - BEHAVIOR CHANGE: Refactor error code retry logic. RPCs should now more + consistently retry specific error codes based on whether they're idempotent + or non-idempotent. +- httpreplay: Fixed a bug when a non-GET request had a zero-length body causing + the Content-Length header to be dropped. +- iot: + - Add new apiv1 client. +- securitycenter: + - Add new apiv1 client. +- cloudscheduler: + - Add new apiv1 client. + +## 0.36.0 + +- spanner: + - Reduce minimum retry backoff from 1s to 100ms. This makes time between + retries much faster and should improve latency. +- storage: + - Add support for Bucket Policy Only. +- kms: + - Add ResourceIAM helper method. + - Deprecate KeyRingIAM and CryptoKeyIAM. Please use ResourceIAM. +- firestore: + - Switch from v1beta1 API to v1 API. + - Allow emulator with FIRESTORE_EMULATOR_HOST. +- bigquery: + - Add NumLongTermBytes to Table. + - Add TotalBytesProcessedAccuracy to QueryStatistics. +- irm: + - Add new v1alpha2 client. +- talent: + - Add new v4beta1 client. +- rpcreplay: + - Fix connection to work with grpc >= 1.17. + - It is now required for an actual gRPC server to be running for Dial to + succeed. + +## 0.35.1 + +- spanner: + - Adds OpenCensus views back to public API. + +## v0.35.0 + +- all: + - Add go.mod and go.sum. + - Switch usage of gax-go to gax-go/v2. +- bigquery: + - Fix bug where time partitioning could not be removed from a table. + - Fix panic that occurred with empty query parameters. +- bttest: + - Fix bug where deleted rows were returned by ReadRows. +- bigtable/emulator: + - Configure max message size to 256 MiB. +- firestore: + - Allow non-transactional queries in transactions. + - Allow StartAt/EndBefore on direct children at any depth. + - QuerySnapshotIterator.Stop may be called in an error state. + - Fix bug the prevented reset of transaction write state in between retries. +- functions/metadata: + - Make Metadata.Resource a pointer. +- logging: + - Make SpanID available in logging.Entry. +- metadata: + - Wrap !200 error code in a typed err. +- profiler: + - Add function to check if function name is within a particular file in the + profile. + - Set parent field in create profile request. + - Return kubernetes client to start cluster, so client can be used to poll + cluster. + - Add function for checking if filename is in profile. +- pubsub: + - Fix bug where messages expired without an initial modack in + synchronous=true mode. + - Receive does not retry ResourceExhausted errors. +- spanner: + - client.Close now cancels existing requests and should be much faster for + large amounts of sessions. + - Correctly allow MinOpened sessions to be spun up. + +## v0.34.0 + +- functions/metadata: + - Switch to using JSON in context. + - Make Resource a value. +- vision: Fix ProductSearch return type. +- datastore: Add an example for how to handle MultiError. + +## v0.33.1 + +- compute: Removes an erroneously added go.mod. +- logging: Populate source location in fromLogEntry. + +## v0.33.0 + +- bttest: + - Add support for apply_label_transformer. +- expr: + - Add expr library. +- firestore: + - Support retrieval of missing documents. +- kms: + - Add IAM methods. +- pubsub: + - Clarify extension documentation. +- scheduler: + - Add v1beta1 client. +- vision: + - Add product search helper. + - Add new product search client. + +## v0.32.0 + +Note: This release is the last to support Go 1.6 and 1.8. + +- bigquery: + - Add support for removing an expiration. + - Ignore NeverExpire in Table.Create. + - Validate table expiration time. +- cbt: + - Add note about not supporting arbitrary bytes. +- datastore: + - Align key checks. +- firestore: + - Return an error when using Start/End without providing values. +- pubsub: + - Add pstest Close method. + - Clarify MaxExtension documentation. +- securitycenter: + - Add v1beta1 client. +- spanner: + - Allow nil in mutations. + - Improve doc of SessionPoolConfig.MaxOpened. + - Increase session deletion timeout from 5s to 15s. + +## v0.31.0 + +- bigtable: + - Group mutations across multiple requests. +- bigquery: + - Link to bigquery troubleshooting errors page in bigquery.Error comment. +- cbt: + - Fix go generate command. + - Document usage of both maxage + maxversions. +- datastore: + - Passing nil keys results in ErrInvalidKey. +- firestore: + - Clarify what Document.DataTo does with untouched struct fields. +- profile: + - Validate service name in agent. +- pubsub: + - Fix deadlock with pstest and ctx.Cancel. + - Fix a possible deadlock in pstest. +- trace: + - Update doc URL with new fragment. + +Special thanks to @fastest963 for going above and beyond helping us to debug +hard-to-reproduce Pub/Sub issues. + +## v0.30.0 + +- spanner: DML support added. See https://godoc.org/cloud.google.com/go/spanner#hdr-DML_and_Partitioned_DML for more information. +- bigtable: bttest supports row sample filter. +- functions: metadata package added for accessing Cloud Functions resource metadata. + +## v0.29.0 + +- bigtable: + - Add retry to all idempotent RPCs. + - cbt supports complex GC policies. + - Emulator supports arbitrary bytes in regex filters. +- firestore: Add ArrayUnion and ArrayRemove. +- logging: Add the ContextFunc option to supply the context used for + asynchronous RPCs. +- profiler: Ignore NotDefinedError when fetching the instance name +- pubsub: + - BEHAVIOR CHANGE: Receive doesn't retry if an RPC returns codes.Cancelled. + - BEHAVIOR CHANGE: Receive retries on Unavailable intead of returning. + - Fix deadlock. + - Restore Ack/Nack/Modacks metrics. + - Improve context handling in iterator. + - Implement synchronous mode for Receive. + - pstest: add Pull. +- spanner: Add a metric for the number of sessions currently opened. +- storage: + - Canceling the context releases all resources. + - Add additional RetentionPolicy attributes. +- vision/apiv1: Add LocalizeObjects method. + +## v0.28.0 + +- bigtable: + - Emulator returns Unimplemented for snapshot RPCs. +- bigquery: + - Support zero-length repeated, nested fields. +- cloud assets: + - Add v1beta client. +- datastore: + - Don't nil out transaction ID on retry. +- firestore: + - BREAKING CHANGE: When watching a query with Query.Snapshots, QuerySnapshotIterator.Next + returns a QuerySnapshot which contains read time, result size, change list and the DocumentIterator + (previously, QuerySnapshotIterator.Next returned just the DocumentIterator). See: https://godoc.org/cloud.google.com/go/firestore#Query.Snapshots. + - Add array-contains operator. +- IAM: + - Add iam/credentials/apiv1 client. +- pubsub: + - Canceling the context passed to Subscription.Receive causes Receive to return when + processing finishes on all messages currently in progress, even if new messages are arriving. +- redis: + - Add redis/apiv1 client. +- storage: + - Add Reader.Attrs. + - Deprecate several Reader getter methods: please use Reader.Attrs for these instead. + - Add ObjectHandle.Bucket and ObjectHandle.Object methods. + +## v0.27.0 + +- bigquery: + - Allow modification of encryption configuration and partitioning options to a table via the Update call. + - Add a SchemaFromJSON function that converts a JSON table schema. +- bigtable: + - Restore cbt count functionality. +- containeranalysis: + - Add v1beta client. +- spanner: + - Fix a case where an iterator might not be closed correctly. +- storage: + - Add ServiceAccount method https://godoc.org/cloud.google.com/go/storage#Client.ServiceAccount. + - Add a method to Reader that returns the parsed value of the Last-Modified header. + +## v0.26.0 + +- bigquery: + - Support filtering listed jobs by min/max creation time. + - Support data clustering (https://godoc.org/cloud.google.com/go/bigquery#Clustering). + - Include job creator email in Job struct. +- bigtable: + - Add `RowSampleFilter`. + - emulator: BREAKING BEHAVIOR CHANGE: Regexps in row, family, column and value filters + must match the entire target string to succeed. Previously, the emulator was + succeeding on partial matches. + NOTE: As of this release, this change only affects the emulator when run + from this repo (bigtable/cmd/emulator/cbtemulator.go). The version launched + from `gcloud` will be updated in a subsequent `gcloud` release. +- dataproc: Add apiv1beta2 client. +- datastore: Save non-nil pointer fields on omitempty. +- logging: populate Entry.Trace from the HTTP X-Cloud-Trace-Context header. +- logging/logadmin: Support writer_identity and include_children. +- pubsub: + - Support labels on topics and subscriptions. + - Support message storage policy for topics. + - Use the distribution of ack times to determine when to extend ack deadlines. + The only user-visible effect of this change should be that programs that + call only `Subscription.Receive` need no IAM permissions other than `Pub/Sub + Subscriber`. +- storage: + - Support predefined ACLs. + - Support additional ACL fields other than Entity and Role. + - Support bucket websites. + - Support bucket logging. + + +## v0.25.0 + +- Added [Code of Conduct](https://github.com/googleapis/google-cloud-go/blob/master/CODE_OF_CONDUCT.md) +- bigtable: + - cbt: Support a GC policy of "never". +- errorreporting: + - Support User. + - Close now calls Flush. + - Use OnError (previously ignored). + - Pass through the RPC error as-is to OnError. +- httpreplay: A tool for recording and replaying HTTP requests + (for the bigquery and storage clients in this repo). +- kms: v1 client added +- logging: add SourceLocation to Entry. +- storage: improve CRC checking on read. + +## v0.24.0 + +- bigquery: Support for the NUMERIC type. +- bigtable: + - cbt: Optionally specify columns for read/lookup + - Support instance-level administration. +- oslogin: New client for the OS Login API. +- pubsub: + - The package is now stable. There will be no further breaking changes. + - Internal changes to improve Subscription.Receive behavior. +- storage: Support updating bucket lifecycle config. +- spanner: Support struct-typed parameter bindings. +- texttospeech: New client for the Text-to-Speech API. + +## v0.23.0 + +- bigquery: Add DDL stats to query statistics. +- bigtable: + - cbt: Add cells-per-column limit for row lookup. + - cbt: Make it possible to combine read filters. +- dlp: v2beta2 client removed. Use the v2 client instead. +- firestore, spanner: Fix compilation errors due to protobuf changes. + +## v0.22.0 + +- bigtable: + - cbt: Support cells per column limit for row read. + - bttest: Correctly handle empty RowSet. + - Fix ReadModifyWrite operation in emulator. + - Fix API path in GetCluster. + +- bigquery: + - BEHAVIOR CHANGE: Retry on 503 status code. + - Add dataset.DeleteWithContents. + - Add SchemaUpdateOptions for query jobs. + - Add Timeline to QueryStatistics. + - Add more stats to ExplainQueryStage. + - Support Parquet data format. + +- datastore: + - Support omitempty for times. + +- dlp: + - **BREAKING CHANGE:** Remove v1beta1 client. Please migrate to the v2 client, + which is now out of beta. + - Add v2 client. + +- firestore: + - BEHAVIOR CHANGE: Treat set({}, MergeAll) as valid. + +- iam: + - Support JWT signing via SignJwt callopt. + +- profiler: + - BEHAVIOR CHANGE: PollForSerialOutput returns an error when context.Done. + - BEHAVIOR CHANGE: Increase the initial backoff to 1 minute. + - Avoid returning empty serial port output. + +- pubsub: + - BEHAVIOR CHANGE: Don't backoff during next retryable error once stream is healthy. + - BEHAVIOR CHANGE: Don't backoff on EOF. + - pstest: Support Acknowledge and ModifyAckDeadline RPCs. + +- redis: + - Add v1 beta Redis client. + +- spanner: + - Support SessionLabels. + +- speech: + - Add api v1 beta1 client. + +- storage: + - BEHAVIOR CHANGE: Retry reads when retryable error occurs. + - Fix delete of object in requester-pays bucket. + - Support KMS integration. + +## v0.21.0 + +- bigquery: + - Add OpenCensus tracing. + +- firestore: + - **BREAKING CHANGE:** If a document does not exist, return a DocumentSnapshot + whose Exists method returns false. DocumentRef.Get and Transaction.Get + return the non-nil DocumentSnapshot in addition to a NotFound error. + **DocumentRef.GetAll and Transaction.GetAll return a non-nil + DocumentSnapshot instead of nil.** + - Add DocumentIterator.Stop. **Call Stop whenever you are done with a + DocumentIterator.** + - Added Query.Snapshots and DocumentRef.Snapshots, which provide realtime + notification of updates. See https://cloud.google.com/firestore/docs/query-data/listen. + - Canceling an RPC now always returns a grpc.Status with codes.Canceled. + +- spanner: + - Add `CommitTimestamp`, which supports inserting the commit timestamp of a + transaction into a column. + +## v0.20.0 + +- bigquery: Support SchemaUpdateOptions for load jobs. + +- bigtable: + - Add SampleRowKeys. + - cbt: Support union, intersection GCPolicy. + - Retry admin RPCS. + - Add trace spans to retries. + +- datastore: Add OpenCensus tracing. + +- firestore: + - Fix queries involving Null and NaN. + - Allow Timestamp protobuffers for time values. + +- logging: Add a WriteTimeout option. + +- spanner: Support Batch API. + +- storage: Add OpenCensus tracing. + +## v0.19.0 + +- bigquery: + - Support customer-managed encryption keys. + +- bigtable: + - Improved emulator support. + - Support GetCluster. + +- datastore: + - Add general mutations. + - Support pointer struct fields. + - Support transaction options. + +- firestore: + - Add Transaction.GetAll. + - Support document cursors. + +- logging: + - Support concurrent RPCs to the service. + - Support per-entry resources. + +- profiler: + - Add config options to disable heap and thread profiling. + - Read the project ID from $GOOGLE_CLOUD_PROJECT when it's set. + +- pubsub: + - BEHAVIOR CHANGE: Release flow control after ack/nack (instead of after the + callback returns). + - Add SubscriptionInProject. + - Add OpenCensus instrumentation for streaming pull. + +- storage: + - Support CORS. + +## v0.18.0 + +- bigquery: + - Marked stable. + - Schema inference of nullable fields supported. + - Added TimePartitioning to QueryConfig. + +- firestore: Data provided to DocumentRef.Set with a Merge option can contain + Delete sentinels. + +- logging: Clients can accept parent resources other than projects. + +- pubsub: + - pubsub/pstest: A lighweight fake for pubsub. Experimental; feedback welcome. + - Support updating more subscription metadata: AckDeadline, + RetainAckedMessages and RetentionDuration. + +- oslogin/apiv1beta: New client for the Cloud OS Login API. + +- rpcreplay: A package for recording and replaying gRPC traffic. + +- spanner: + - Add a ReadWithOptions that supports a row limit, as well as an index. + - Support query plan and execution statistics. + - Added [OpenCensus](http://opencensus.io) support. + +- storage: Clarify checksum validation for gzipped files (it is not validated + when the file is served uncompressed). + + +## v0.17.0 + +- firestore BREAKING CHANGES: + - Remove UpdateMap and UpdateStruct; rename UpdatePaths to Update. + Change + `docref.UpdateMap(ctx, map[string]interface{}{"a.b", 1})` + to + `docref.Update(ctx, []firestore.Update{{Path: "a.b", Value: 1}})` + + Change + `docref.UpdateStruct(ctx, []string{"Field"}, aStruct)` + to + `docref.Update(ctx, []firestore.Update{{Path: "Field", Value: aStruct.Field}})` + - Rename MergePaths to Merge; require args to be FieldPaths + - A value stored as an integer can be read into a floating-point field, and vice versa. +- bigtable/cmd/cbt: + - Support deleting a column. + - Add regex option for row read. +- spanner: Mark stable. +- storage: + - Add Reader.ContentEncoding method. + - Fix handling of SignedURL headers. +- bigquery: + - If Uploader.Put is called with no rows, it returns nil without making a + call. + - Schema inference supports the "nullable" option in struct tags for + non-required fields. + - TimePartitioning supports "Field". + + +## v0.16.0 + +- Other bigquery changes: + - `JobIterator.Next` returns `*Job`; removed `JobInfo` (BREAKING CHANGE). + - UseStandardSQL is deprecated; set UseLegacySQL to true if you need + Legacy SQL. + - Uploader.Put will generate a random insert ID if you do not provide one. + - Support time partitioning for load jobs. + - Support dry-run queries. + - A `Job` remembers its last retrieved status. + - Support retrieving job configuration. + - Support labels for jobs and tables. + - Support dataset access lists. + - Improve support for external data sources, including data from Bigtable and + Google Sheets, and tables with external data. + - Support updating a table's view configuration. + - Fix uploading civil times with nanoseconds. + +- storage: + - Support PubSub notifications. + - Support Requester Pays buckets. + +- profiler: Support goroutine and mutex profile types. + +## v0.15.0 + +- firestore: beta release. See the + [announcement](https://firebase.googleblog.com/2017/10/introducing-cloud-firestore.html). + +- errorreporting: The existing package has been redesigned. + +- errors: This package has been removed. Use errorreporting. + + +## v0.14.0 + +- bigquery BREAKING CHANGES: + - Standard SQL is the default for queries and views. + - `Table.Create` takes `TableMetadata` as a second argument, instead of + options. + - `Dataset.Create` takes `DatasetMetadata` as a second argument. + - `DatasetMetadata` field `ID` renamed to `FullID` + - `TableMetadata` field `ID` renamed to `FullID` + +- Other bigquery changes: + - The client will append a random suffix to a provided job ID if you set + `AddJobIDSuffix` to true in a job config. + - Listing jobs is supported. + - Better retry logic. + +- vision, language, speech: clients are now stable + +- monitoring: client is now beta + +- profiler: + - Rename InstanceName to Instance, ZoneName to Zone + - Auto-detect service name and version on AppEngine. + +## v0.13.0 + +- bigquery: UseLegacySQL options for CreateTable and QueryConfig. Use these + options to continue using Legacy SQL after the client switches its default + to Standard SQL. + +- bigquery: Support for updating dataset labels. + +- bigquery: Set DatasetIterator.ProjectID to list datasets in a project other + than the client's. DatasetsInProject is no longer needed and is deprecated. + +- bigtable: Fail ListInstances when any zones fail. + +- spanner: support decoding of slices of basic types (e.g. []string, []int64, + etc.) + +- logging/logadmin: UpdateSink no longer creates a sink if it is missing + (actually a change to the underlying service, not the client) + +- profiler: Service and ServiceVersion replace Target in Config. + +## v0.12.0 + +- pubsub: Subscription.Receive now uses streaming pull. + +- pubsub: add Client.TopicInProject to access topics in a different project + than the client. + +- errors: renamed errorreporting. The errors package will be removed shortly. + +- datastore: improved retry behavior. + +- bigquery: support updates to dataset metadata, with etags. + +- bigquery: add etag support to Table.Update (BREAKING: etag argument added). + +- bigquery: generate all job IDs on the client. + +- storage: support bucket lifecycle configurations. + + +## v0.11.0 + +- Clients for spanner, pubsub and video are now in beta. + +- New client for DLP. + +- spanner: performance and testing improvements. + +- storage: requester-pays buckets are supported. + +- storage, profiler, bigtable, bigquery: bug fixes and other minor improvements. + +- pubsub: bug fixes and other minor improvements + +## v0.10.0 + +- pubsub: Subscription.ModifyPushConfig replaced with Subscription.Update. + +- pubsub: Subscription.Receive now runs concurrently for higher throughput. + +- vision: cloud.google.com/go/vision is deprecated. Use +cloud.google.com/go/vision/apiv1 instead. + +- translation: now stable. + +- trace: several changes to the surface. See the link below. + +### Code changes required from v0.9.0 + +- pubsub: Replace + + ``` + sub.ModifyPushConfig(ctx, pubsub.PushConfig{Endpoint: "https://example.com/push"}) + ``` + + with + + ``` + sub.Update(ctx, pubsub.SubscriptionConfigToUpdate{ + PushConfig: &pubsub.PushConfig{Endpoint: "https://example.com/push"}, + }) + ``` + +- trace: traceGRPCServerInterceptor will be provided from *trace.Client. +Given an initialized `*trace.Client` named `tc`, instead of + + ``` + s := grpc.NewServer(grpc.UnaryInterceptor(trace.GRPCServerInterceptor(tc))) + ``` + + write + + ``` + s := grpc.NewServer(grpc.UnaryInterceptor(tc.GRPCServerInterceptor())) + ``` + +- trace trace.GRPCClientInterceptor will also provided from *trace.Client. +Instead of + + ``` + conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(trace.GRPCClientInterceptor())) + ``` + + write + + ``` + conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor())) + ``` + +- trace: We removed the deprecated `trace.EnableGRPCTracing`. Use the gRPC +interceptor as a dial option as shown below when initializing Cloud package +clients: + + ``` + c, err := pubsub.NewClient(ctx, "project-id", option.WithGRPCDialOption(grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor()))) + if err != nil { + ... + } + ``` + + +## v0.9.0 + +- Breaking changes to some autogenerated clients. +- rpcreplay package added. + +## v0.8.0 + +- profiler package added. +- storage: + - Retry Objects.Insert call. + - Add ProgressFunc to WRiter. +- pubsub: breaking changes: + - Publish is now asynchronous ([announcement](https://groups.google.com/d/topic/google-api-go-announce/aaqRDIQ3rvU/discussion)). + - Subscription.Pull replaced by Subscription.Receive, which takes a callback ([announcement](https://groups.google.com/d/topic/google-api-go-announce/8pt6oetAdKc/discussion)). + - Message.Done replaced with Message.Ack and Message.Nack. + +## v0.7.0 + +- Release of a client library for Spanner. See +the +[blog +post](https://cloudplatform.googleblog.com/2017/02/introducing-Cloud-Spanner-a-global-database-service-for-mission-critical-applications.html). +Note that although the Spanner service is beta, the Go client library is alpha. + +## v0.6.0 + +- Beta release of BigQuery, DataStore, Logging and Storage. See the +[blog post](https://cloudplatform.googleblog.com/2016/12/announcing-new-google-cloud-client.html). + +- bigquery: + - struct support. Read a row directly into a struct with +`RowIterator.Next`, and upload a row directly from a struct with `Uploader.Put`. +You can also use field tags. See the [package documentation][cloud-bigquery-ref] +for details. + + - The `ValueList` type was removed. It is no longer necessary. Instead of + ```go + var v ValueList + ... it.Next(&v) .. + ``` + use + + ```go + var v []Value + ... it.Next(&v) ... + ``` + + - Previously, repeatedly calling `RowIterator.Next` on the same `[]Value` or + `ValueList` would append to the slice. Now each call resets the size to zero first. + + - Schema inference will infer the SQL type BYTES for a struct field of + type []byte. Previously it inferred STRING. + + - The types `uint`, `uint64` and `uintptr` are no longer supported in schema + inference. BigQuery's integer type is INT64, and those types may hold values + that are not correctly represented in a 64-bit signed integer. + +## v0.5.0 + +- bigquery: + - The SQL types DATE, TIME and DATETIME are now supported. They correspond to + the `Date`, `Time` and `DateTime` types in the new `cloud.google.com/go/civil` + package. + - Support for query parameters. + - Support deleting a dataset. + - Values from INTEGER columns will now be returned as int64, not int. This + will avoid errors arising from large values on 32-bit systems. +- datastore: + - Nested Go structs encoded as Entity values, instead of a +flattened list of the embedded struct's fields. This means that you may now have twice-nested slices, eg. + ```go + type State struct { + Cities []struct{ + Populations []int + } + } + ``` + See [the announcement](https://groups.google.com/forum/#!topic/google-api-go-announce/79jtrdeuJAg) for +more details. + - Contexts no longer hold namespaces; instead you must set a key's namespace + explicitly. Also, key functions have been changed and renamed. + - The WithNamespace function has been removed. To specify a namespace in a Query, use the Query.Namespace method: + ```go + q := datastore.NewQuery("Kind").Namespace("ns") + ``` + - All the fields of Key are exported. That means you can construct any Key with a struct literal: + ```go + k := &Key{Kind: "Kind", ID: 37, Namespace: "ns"} + ``` + - As a result of the above, the Key methods Kind, ID, d.Name, Parent, SetParent and Namespace have been removed. + - `NewIncompleteKey` has been removed, replaced by `IncompleteKey`. Replace + ```go + NewIncompleteKey(ctx, kind, parent) + ``` + with + ```go + IncompleteKey(kind, parent) + ``` + and if you do use namespaces, make sure you set the namespace on the returned key. + - `NewKey` has been removed, replaced by `NameKey` and `IDKey`. Replace + ```go + NewKey(ctx, kind, name, 0, parent) + NewKey(ctx, kind, "", id, parent) + ``` + with + ```go + NameKey(kind, name, parent) + IDKey(kind, id, parent) + ``` + and if you do use namespaces, make sure you set the namespace on the returned key. + - The `Done` variable has been removed. Replace `datastore.Done` with `iterator.Done`, from the package `google.golang.org/api/iterator`. + - The `Client.Close` method will have a return type of error. It will return the result of closing the underlying gRPC connection. + - See [the announcement](https://groups.google.com/forum/#!topic/google-api-go-announce/hqXtM_4Ix-0) for +more details. + +## v0.4.0 + +- bigquery: + -`NewGCSReference` is now a function, not a method on `Client`. + - `Table.LoaderFrom` now accepts a `ReaderSource`, enabling + loading data into a table from a file or any `io.Reader`. + * Client.Table and Client.OpenTable have been removed. + Replace + ```go + client.OpenTable("project", "dataset", "table") + ``` + with + ```go + client.DatasetInProject("project", "dataset").Table("table") + ``` + + * Client.CreateTable has been removed. + Replace + ```go + client.CreateTable(ctx, "project", "dataset", "table") + ``` + with + ```go + client.DatasetInProject("project", "dataset").Table("table").Create(ctx) + ``` + + * Dataset.ListTables have been replaced with Dataset.Tables. + Replace + ```go + tables, err := ds.ListTables(ctx) + ``` + with + ```go + it := ds.Tables(ctx) + for { + table, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + // TODO: Handle error. + } + // TODO: use table. + } + ``` + + * Client.Read has been replaced with Job.Read, Table.Read and Query.Read. + Replace + ```go + it, err := client.Read(ctx, job) + ``` + with + ```go + it, err := job.Read(ctx) + ``` + and similarly for reading from tables or queries. + + * The iterator returned from the Read methods is now named RowIterator. Its + behavior is closer to the other iterators in these libraries. It no longer + supports the Schema method; see the next item. + Replace + ```go + for it.Next(ctx) { + var vals ValueList + if err := it.Get(&vals); err != nil { + // TODO: Handle error. + } + // TODO: use vals. + } + if err := it.Err(); err != nil { + // TODO: Handle error. + } + ``` + with + ``` + for { + var vals ValueList + err := it.Next(&vals) + if err == iterator.Done { + break + } + if err != nil { + // TODO: Handle error. + } + // TODO: use vals. + } + ``` + Instead of the `RecordsPerRequest(n)` option, write + ```go + it.PageInfo().MaxSize = n + ``` + Instead of the `StartIndex(i)` option, write + ```go + it.StartIndex = i + ``` + + * ValueLoader.Load now takes a Schema in addition to a slice of Values. + Replace + ```go + func (vl *myValueLoader) Load(v []bigquery.Value) + ``` + with + ```go + func (vl *myValueLoader) Load(v []bigquery.Value, s bigquery.Schema) + ``` + + + * Table.Patch is replace by Table.Update. + Replace + ```go + p := table.Patch() + p.Description("new description") + metadata, err := p.Apply(ctx) + ``` + with + ```go + metadata, err := table.Update(ctx, bigquery.TableMetadataToUpdate{ + Description: "new description", + }) + ``` + + * Client.Copy is replaced by separate methods for each of its four functions. + All options have been replaced by struct fields. + + * To load data from Google Cloud Storage into a table, use Table.LoaderFrom. + + Replace + ```go + client.Copy(ctx, table, gcsRef) + ``` + with + ```go + table.LoaderFrom(gcsRef).Run(ctx) + ``` + Instead of passing options to Copy, set fields on the Loader: + ```go + loader := table.LoaderFrom(gcsRef) + loader.WriteDisposition = bigquery.WriteTruncate + ``` + + * To extract data from a table into Google Cloud Storage, use + Table.ExtractorTo. Set fields on the returned Extractor instead of + passing options. + + Replace + ```go + client.Copy(ctx, gcsRef, table) + ``` + with + ```go + table.ExtractorTo(gcsRef).Run(ctx) + ``` + + * To copy data into a table from one or more other tables, use + Table.CopierFrom. Set fields on the returned Copier instead of passing options. + + Replace + ```go + client.Copy(ctx, dstTable, srcTable) + ``` + with + ```go + dst.Table.CopierFrom(srcTable).Run(ctx) + ``` + + * To start a query job, create a Query and call its Run method. Set fields + on the query instead of passing options. + + Replace + ```go + client.Copy(ctx, table, query) + ``` + with + ```go + query.Run(ctx) + ``` + + * Table.NewUploader has been renamed to Table.Uploader. Instead of options, + configure an Uploader by setting its fields. + Replace + ```go + u := table.NewUploader(bigquery.UploadIgnoreUnknownValues()) + ``` + with + ```go + u := table.NewUploader(bigquery.UploadIgnoreUnknownValues()) + u.IgnoreUnknownValues = true + ``` + +- pubsub: remove `pubsub.Done`. Use `iterator.Done` instead, where `iterator` is the package +`google.golang.org/api/iterator`. + +## v0.3.0 + +- storage: + * AdminClient replaced by methods on Client. + Replace + ```go + adminClient.CreateBucket(ctx, bucketName, attrs) + ``` + with + ```go + client.Bucket(bucketName).Create(ctx, projectID, attrs) + ``` + + * BucketHandle.List replaced by BucketHandle.Objects. + Replace + ```go + for query != nil { + objs, err := bucket.List(d.ctx, query) + if err != nil { ... } + query = objs.Next + for _, obj := range objs.Results { + fmt.Println(obj) + } + } + ``` + with + ```go + iter := bucket.Objects(d.ctx, query) + for { + obj, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { ... } + fmt.Println(obj) + } + ``` + (The `iterator` package is at `google.golang.org/api/iterator`.) + + Replace `Query.Cursor` with `ObjectIterator.PageInfo().Token`. + + Replace `Query.MaxResults` with `ObjectIterator.PageInfo().MaxSize`. + + + * ObjectHandle.CopyTo replaced by ObjectHandle.CopierFrom. + Replace + ```go + attrs, err := src.CopyTo(ctx, dst, nil) + ``` + with + ```go + attrs, err := dst.CopierFrom(src).Run(ctx) + ``` + + Replace + ```go + attrs, err := src.CopyTo(ctx, dst, &storage.ObjectAttrs{ContextType: "text/html"}) + ``` + with + ```go + c := dst.CopierFrom(src) + c.ContextType = "text/html" + attrs, err := c.Run(ctx) + ``` + + * ObjectHandle.ComposeFrom replaced by ObjectHandle.ComposerFrom. + Replace + ```go + attrs, err := dst.ComposeFrom(ctx, []*storage.ObjectHandle{src1, src2}, nil) + ``` + with + ```go + attrs, err := dst.ComposerFrom(src1, src2).Run(ctx) + ``` + + * ObjectHandle.Update's ObjectAttrs argument replaced by ObjectAttrsToUpdate. + Replace + ```go + attrs, err := obj.Update(ctx, &storage.ObjectAttrs{ContextType: "text/html"}) + ``` + with + ```go + attrs, err := obj.Update(ctx, storage.ObjectAttrsToUpdate{ContextType: "text/html"}) + ``` + + * ObjectHandle.WithConditions replaced by ObjectHandle.If. + Replace + ```go + obj.WithConditions(storage.Generation(gen), storage.IfMetaGenerationMatch(mgen)) + ``` + with + ```go + obj.Generation(gen).If(storage.Conditions{MetagenerationMatch: mgen}) + ``` + + Replace + ```go + obj.WithConditions(storage.IfGenerationMatch(0)) + ``` + with + ```go + obj.If(storage.Conditions{DoesNotExist: true}) + ``` + + * `storage.Done` replaced by `iterator.Done` (from package `google.golang.org/api/iterator`). + +- Package preview/logging deleted. Use logging instead. + +## v0.2.0 + +- Logging client replaced with preview version (see below). + +- New clients for some of Google's Machine Learning APIs: Vision, Speech, and +Natural Language. + +- Preview version of a new [Stackdriver Logging][cloud-logging] client in +[`cloud.google.com/go/preview/logging`](https://godoc.org/cloud.google.com/go/preview/logging). +This client uses gRPC as its transport layer, and supports log reading, sinks +and metrics. It will replace the current client at `cloud.google.com/go/logging` shortly. diff --git a/vendor/cloud.google.com/go/CODE_OF_CONDUCT.md b/vendor/cloud.google.com/go/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..8fd1bc9c2 --- /dev/null +++ b/vendor/cloud.google.com/go/CODE_OF_CONDUCT.md @@ -0,0 +1,44 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, +and in the interest of fostering an open and welcoming community, +we pledge to respect all people who contribute through reporting issues, +posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project +a harassment-free experience for everyone, +regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, +such as physical or electronic +addresses, without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct. +By adopting this Code of Conduct, +project maintainers commit themselves to fairly and consistently +applying these principles to every aspect of managing this project. +Project maintainers who do not follow or enforce the Code of Conduct +may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by opening an issue +or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0, +available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/) + diff --git a/vendor/cloud.google.com/go/CONTRIBUTING.md b/vendor/cloud.google.com/go/CONTRIBUTING.md new file mode 100644 index 000000000..6d6e48b65 --- /dev/null +++ b/vendor/cloud.google.com/go/CONTRIBUTING.md @@ -0,0 +1,327 @@ +# Contributing + +1. [File an issue](https://github.com/googleapis/google-cloud-go/issues/new/choose). + The issue will be used to discuss the bug or feature and should be created + before sending a PR. + +1. [Install Go](https://golang.org/dl/). + 1. Ensure that your `GOBIN` directory (by default `$(go env GOPATH)/bin`) + is in your `PATH`. + 1. Check it's working by running `go version`. + * If it doesn't work, check the install location, usually + `/usr/local/go`, is on your `PATH`. + +1. Sign one of the +[contributor license agreements](#contributor-license-agreements) below. + +1. Clone the repo: + `git clone https://github.com/googleapis/google-cloud-go` + +1. Change into the checked out source: + `cd google-cloud-go` + +1. Fork the repo. + +1. Set your fork as a remote: + `git remote add fork git@github.com:GITHUB_USERNAME/google-cloud-go.git` + +1. Make changes, commit to your fork. + + Commit messages should follow the + [Conventional Commits Style](https://www.conventionalcommits.org). The scope + portion should always be filled with the name of the package affected by the + changes being made. For example: + ``` + feat(functions): add gophers codelab + ``` + +1. Send a pull request with your changes. + + To minimize friction, consider setting `Allow edits from maintainers` on the + PR, which will enable project committers and automation to update your PR. + +1. A maintainer will review the pull request and make comments. + + Prefer adding additional commits over amending and force-pushing since it can + be difficult to follow code reviews when the commit history changes. + + Commits will be squashed when they're merged. + +## Testing + +We test code against two versions of Go, the minimum and maximum versions +supported by our clients. To see which versions these are checkout our +[README](README.md#supported-versions). + +### Integration Tests + +In addition to the unit tests, you may run the integration test suite. These +directions describe setting up your environment to run integration tests for +_all_ packages: note that many of these instructions may be redundant if you +intend only to run integration tests on a single package. + +#### GCP Setup + +To run the integrations tests, creation and configuration of two projects in +the Google Developers Console is required: one specifically for Firestore +integration tests, and another for all other integration tests. We'll refer to +these projects as "general project" and "Firestore project". + +After creating each project, you must [create a service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount) +for each project. Ensure the project-level **Owner** +[IAM role](https://console.cloud.google.com/iam-admin/iam/project) role is added to +each service account. During the creation of the service account, you should +download the JSON credential file for use later. + +Next, ensure the following APIs are enabled in the general project: + +- BigQuery API +- BigQuery Data Transfer API +- Cloud Dataproc API +- Cloud Dataproc Control API Private +- Cloud Datastore API +- Cloud Firestore API +- Cloud Key Management Service (KMS) API +- Cloud Natural Language API +- Cloud OS Login API +- Cloud Pub/Sub API +- Cloud Resource Manager API +- Cloud Spanner API +- Cloud Speech API +- Cloud Translation API +- Cloud Video Intelligence API +- Cloud Vision API +- Compute Engine API +- Compute Engine Instance Group Manager API +- Container Registry API +- Firebase Rules API +- Google Cloud APIs +- Google Cloud Deployment Manager V2 API +- Google Cloud SQL +- Google Cloud Storage +- Google Cloud Storage JSON API +- Google Compute Engine Instance Group Updater API +- Google Compute Engine Instance Groups API +- Kubernetes Engine API +- Cloud Error Reporting API +- Pub/Sub Lite API + +Next, create a Datastore database in the general project, and a Firestore +database in the Firestore project. + +Finally, in the general project, create an API key for the translate API: + +- Go to GCP Developer Console. +- Navigate to APIs & Services > Credentials. +- Click Create Credentials > API Key. +- Save this key for use in `GCLOUD_TESTS_API_KEY` as described below. + +#### Local Setup + +Once the two projects are created and configured, set the following environment +variables: + +- `GCLOUD_TESTS_GOLANG_PROJECT_ID`: Developers Console project's ID (e.g. +bamboo-shift-455) for the general project. +- `GCLOUD_TESTS_GOLANG_KEY`: The path to the JSON key file of the general +project's service account. +- `GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID`: Developers Console project's ID +(e.g. doorway-cliff-677) for the Firestore project. +- `GCLOUD_TESTS_GOLANG_FIRESTORE_KEY`: The path to the JSON key file of the +Firestore project's service account. +- `GCLOUD_TESTS_API_KEY`: API key for using the Translate API created above. + +As part of the setup that follows, the following variables will be configured: + +- `GCLOUD_TESTS_GOLANG_KEYRING`: The full name of the keyring for the tests, +in the form +"projects/P/locations/L/keyRings/R". The creation of this is described below. +- `GCLOUD_TESTS_BIGTABLE_KEYRING`: The full name of the keyring for the bigtable tests, +in the form +"projects/P/locations/L/keyRings/R". The creation of this is described below. Expected to be single region. +- `GCLOUD_TESTS_GOLANG_ZONE`: Compute Engine zone. + +Install the [gcloud command-line tool][gcloudcli] to your machine and use it to +create some resources used in integration tests. + +From the project's root directory: + +``` sh +# Sets the default project in your env. +$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID + +# Authenticates the gcloud tool with your account. +$ gcloud auth login + +# Create the indexes used in the datastore integration tests. +$ gcloud datastore indexes create datastore/testdata/index.yaml + +# Creates a Google Cloud storage bucket with the same name as your test project, +# and with the Cloud Logging service account as owner, for the sink +# integration tests in logging. +$ gsutil mb gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID +$ gsutil acl ch -g cloud-logs@google.com:O gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID + +# Creates a PubSub topic for integration tests of storage notifications. +$ gcloud beta pubsub topics create go-storage-notification-test +# Next, go to the Pub/Sub dashboard in GCP console. Authorize the user +# "service-@gs-project-accounts.iam.gserviceaccount.com" +# as a publisher to that topic. + +# Creates a Spanner instance for the spanner integration tests. +$ gcloud beta spanner instances create go-integration-test --config regional-us-central1 --nodes 10 --description 'Instance for go client test' +# NOTE: Spanner instances are priced by the node-hour, so you may want to +# delete the instance after testing with 'gcloud beta spanner instances delete'. + +$ export MY_KEYRING=some-keyring-name +$ export MY_LOCATION=global +$ export MY_SINGLE_LOCATION=us-central1 +# Creates a KMS keyring, in the same location as the default location for your +# project's buckets. +$ gcloud kms keyrings create $MY_KEYRING --location $MY_LOCATION +# Creates two keys in the keyring, named key1 and key2. +$ gcloud kms keys create key1 --keyring $MY_KEYRING --location $MY_LOCATION --purpose encryption +$ gcloud kms keys create key2 --keyring $MY_KEYRING --location $MY_LOCATION --purpose encryption +# Sets the GCLOUD_TESTS_GOLANG_KEYRING environment variable. +$ export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_LOCATION/keyRings/$MY_KEYRING +# Authorizes Google Cloud Storage to encrypt and decrypt using key1. +$ gsutil kms authorize -p $GCLOUD_TESTS_GOLANG_PROJECT_ID -k $GCLOUD_TESTS_GOLANG_KEYRING/cryptoKeys/key1 + +# Create KMS Key in one region for Bigtable +$ gcloud kms keyrings create $MY_KEYRING --location $MY_SINGLE_LOCATION +$ gcloud kms keys create key1 --keyring $MY_KEYRING --location $MY_SINGLE_LOCATION --purpose encryption +# Sets the GCLOUD_TESTS_BIGTABLE_KEYRING environment variable. +$ export GCLOUD_TESTS_BIGTABLE_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_SINGLE_LOCATION/keyRings/$MY_KEYRING +# Create a service agent, https://cloud.google.com/bigtable/docs/use-cmek#gcloud: +$ gcloud beta services identity create \ + --service=bigtableadmin.googleapis.com \ + --project $GCLOUD_TESTS_GOLANG_PROJECT_ID +# Note the service agent email for the agent created. +$ export SERVICE_AGENT_EMAIL= + +# Authorizes Google Cloud Bigtable to encrypt and decrypt using key1 +$ gcloud kms keys add-iam-policy-binding key1 \ + --keyring $MY_KEYRING \ + --location $MY_SINGLE_LOCATION \ + --role roles/cloudkms.cryptoKeyEncrypterDecrypter \ + --member "serviceAccount:$SERVICE_AGENT_EMAIL" \ + --project $GCLOUD_TESTS_GOLANG_PROJECT_ID +``` + +It may be useful to add exports to your shell initialization for future use. +For instance, in `.zshrc`: + +```sh +#### START GO SDK Test Variables +# Developers Console project's ID (e.g. bamboo-shift-455) for the general project. +export GCLOUD_TESTS_GOLANG_PROJECT_ID=your-project + +# The path to the JSON key file of the general project's service account. +export GCLOUD_TESTS_GOLANG_KEY=~/directory/your-project-abcd1234.json + +# Developers Console project's ID (e.g. doorway-cliff-677) for the Firestore project. +export GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID=your-firestore-project + +# The path to the JSON key file of the Firestore project's service account. +export GCLOUD_TESTS_GOLANG_FIRESTORE_KEY=~/directory/your-firestore-project-abcd1234.json + +# The full name of the keyring for the tests, in the form "projects/P/locations/L/keyRings/R". +# The creation of this is described below. +export MY_KEYRING=my-golang-sdk-test +export MY_LOCATION=global +export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_LOCATION/keyRings/$MY_KEYRING + +# API key for using the Translate API. +export GCLOUD_TESTS_API_KEY=abcdefghijk123456789 + +# Compute Engine zone. (https://cloud.google.com/compute/docs/regions-zones) +export GCLOUD_TESTS_GOLANG_ZONE=your-chosen-region +#### END GO SDK Test Variables +``` + +#### Running + +Once you've done the necessary setup, you can run the integration tests by +running: + +``` sh +$ go test -v ./... +``` + +Note that the above command will not run the tests in other modules. To run +tests on other modules, first navigate to the appropriate +subdirectory. For instance, to run only the tests for datastore: +``` sh +$ cd datastore +$ go test -v ./... +``` + +#### Replay + +Some packages can record the RPCs during integration tests to a file for +subsequent replay. To record, pass the `-record` flag to `go test`. The +recording will be saved to the _package_`.replay` file. To replay integration +tests from a saved recording, the replay file must be present, the `-short` +flag must be passed to `go test`, and the `GCLOUD_TESTS_GOLANG_ENABLE_REPLAY` +environment variable must have a non-empty value. + +## Contributor License Agreements + +Before we can accept your pull requests you'll need to sign a Contributor +License Agreement (CLA): + +- **If you are an individual writing original source code** and **you own the +intellectual property**, then you'll need to sign an [individual CLA][indvcla]. +- **If you work for a company that wants to allow you to contribute your +work**, then you'll need to sign a [corporate CLA][corpcla]. + +You can sign these electronically (just scroll to the bottom). After that, +we'll be able to accept your pull requests. + +## Contributor Code of Conduct + +As contributors and maintainers of this project, +and in the interest of fostering an open and welcoming community, +we pledge to respect all people who contribute through reporting issues, +posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project +a harassment-free experience for everyone, +regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, +such as physical or electronic +addresses, without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct. +By adopting this Code of Conduct, +project maintainers commit themselves to fairly and consistently +applying these principles to every aspect of managing this project. +Project maintainers who do not follow or enforce the Code of Conduct +may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by opening an issue +or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.2.0, +available at [https://contributor-covenant.org/version/1/2/0/](https://contributor-covenant.org/version/1/2/0/) + +[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/ +[indvcla]: https://developers.google.com/open-source/cla/individual +[corpcla]: https://developers.google.com/open-source/cla/corporate diff --git a/vendor/cloud.google.com/go/LICENSE b/vendor/cloud.google.com/go/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/cloud.google.com/go/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/cloud.google.com/go/README.md b/vendor/cloud.google.com/go/README.md new file mode 100644 index 000000000..01453cc69 --- /dev/null +++ b/vendor/cloud.google.com/go/README.md @@ -0,0 +1,139 @@ +# Google Cloud Client Libraries for Go + +[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go.svg)](https://pkg.go.dev/cloud.google.com/go) + +Go packages for [Google Cloud Platform](https://cloud.google.com) services. + +``` go +import "cloud.google.com/go" +``` + +To install the packages on your system, *do not clone the repo*. Instead: + +1. Change to your project directory: + + ```bash + cd /my/cloud/project + ``` +1. Get the package you want to use. Some products have their own module, so it's + best to `go get` the package(s) you want to use: + + ``` + $ go get cloud.google.com/go/firestore # Replace with the package you want to use. + ``` + +**NOTE:** Some of these packages are under development, and may occasionally +make backwards-incompatible changes. + +## Supported APIs + +For an updated list of all of our released APIs please see our +[reference docs](https://cloud.google.com/go/docs/reference). + +## [Go Versions Supported](#supported-versions) + +Our libraries are compatible with at least the three most recent, major Go +releases. They are currently compatible with: + +- Go 1.19 +- Go 1.18 +- Go 1.17 +- Go 1.16 +- Go 1.15 + +## Authorization + +By default, each API will use [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials) +for authorization credentials used in calling the API endpoints. This will allow your +application to run in many environments without requiring explicit configuration. + +[snip]:# (auth) +```go +client, err := storage.NewClient(ctx) +``` + +To authorize using a +[JSON key file](https://cloud.google.com/iam/docs/managing-service-account-keys), +pass +[`option.WithCredentialsFile`](https://pkg.go.dev/google.golang.org/api/option#WithCredentialsFile) +to the `NewClient` function of the desired package. For example: + +[snip]:# (auth-JSON) +```go +client, err := storage.NewClient(ctx, option.WithCredentialsFile("path/to/keyfile.json")) +``` + +You can exert more control over authorization by using the +[`golang.org/x/oauth2`](https://pkg.go.dev/golang.org/x/oauth2) package to +create an `oauth2.TokenSource`. Then pass +[`option.WithTokenSource`](https://pkg.go.dev/google.golang.org/api/option#WithTokenSource) +to the `NewClient` function: +[snip]:# (auth-ts) +```go +tokenSource := ... +client, err := storage.NewClient(ctx, option.WithTokenSource(tokenSource)) +``` + +## Contributing + +Contributions are welcome. Please, see the +[CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. +See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. + +[cloud-asset]: https://cloud.google.com/security-command-center/docs/how-to-asset-inventory +[cloud-automl]: https://cloud.google.com/automl +[cloud-build]: https://cloud.google.com/cloud-build/ +[cloud-bigquery]: https://cloud.google.com/bigquery/ +[cloud-bigtable]: https://cloud.google.com/bigtable/ +[cloud-compute]: https://cloud.google.com/compute +[cloud-container]: https://cloud.google.com/containers/ +[cloud-containeranalysis]: https://cloud.google.com/container-registry/docs/container-analysis +[cloud-dataproc]: https://cloud.google.com/dataproc/ +[cloud-datastore]: https://cloud.google.com/datastore/ +[cloud-dialogflow]: https://cloud.google.com/dialogflow-enterprise/ +[cloud-debugger]: https://cloud.google.com/debugger/ +[cloud-dlp]: https://cloud.google.com/dlp/ +[cloud-errors]: https://cloud.google.com/error-reporting/ +[cloud-firestore]: https://cloud.google.com/firestore/ +[cloud-iam]: https://cloud.google.com/iam/ +[cloud-iot]: https://cloud.google.com/iot-core/ +[cloud-irm]: https://cloud.google.com/incident-response/docs/concepts +[cloud-kms]: https://cloud.google.com/kms/ +[cloud-pubsub]: https://cloud.google.com/pubsub/ +[cloud-pubsublite]: https://cloud.google.com/pubsub/lite +[cloud-storage]: https://cloud.google.com/storage/ +[cloud-language]: https://cloud.google.com/natural-language +[cloud-logging]: https://cloud.google.com/logging/ +[cloud-natural-language]: https://cloud.google.com/natural-language/ +[cloud-memorystore]: https://cloud.google.com/memorystore/ +[cloud-monitoring]: https://cloud.google.com/monitoring/ +[cloud-oslogin]: https://cloud.google.com/compute/docs/oslogin/rest +[cloud-phishingprotection]: https://cloud.google.com/phishing-protection/ +[cloud-securitycenter]: https://cloud.google.com/security-command-center/ +[cloud-scheduler]: https://cloud.google.com/scheduler +[cloud-spanner]: https://cloud.google.com/spanner/ +[cloud-speech]: https://cloud.google.com/speech +[cloud-talent]: https://cloud.google.com/solutions/talent-solution/ +[cloud-tasks]: https://cloud.google.com/tasks/ +[cloud-texttospeech]: https://cloud.google.com/texttospeech/ +[cloud-talent]: https://cloud.google.com/solutions/talent-solution/ +[cloud-trace]: https://cloud.google.com/trace/ +[cloud-translate]: https://cloud.google.com/translate +[cloud-recaptcha]: https://cloud.google.com/recaptcha-enterprise/ +[cloud-recommender]: https://cloud.google.com/recommendations/ +[cloud-video]: https://cloud.google.com/video-intelligence/ +[cloud-vision]: https://cloud.google.com/vision +[cloud-webrisk]: https://cloud.google.com/web-risk/ + +## Links + +- [Go on Google Cloud](https://cloud.google.com/go/home) +- [Getting started with Go on Google Cloud](https://cloud.google.com/go/getting-started) +- [App Engine Quickstart](https://cloud.google.com/appengine/docs/standard/go/quickstart) +- [Cloud Functions Quickstart](https://cloud.google.com/functions/docs/quickstart-go) +- [Cloud Run Quickstart](https://cloud.google.com/run/docs/quickstarts/build-and-deploy#go) diff --git a/vendor/cloud.google.com/go/RELEASING.md b/vendor/cloud.google.com/go/RELEASING.md new file mode 100644 index 000000000..6d0fcf4f9 --- /dev/null +++ b/vendor/cloud.google.com/go/RELEASING.md @@ -0,0 +1,141 @@ +# Releasing + +## Determine which module to release + +The Go client libraries have several modules. Each module does not strictly +correspond to a single library - they correspond to trees of directories. If a +file needs to be released, you must release the closest ancestor module. + +To see all modules: + +```bash +$ cat `find . -name go.mod` | grep module +module cloud.google.com/go/pubsub +module cloud.google.com/go/spanner +module cloud.google.com/go +module cloud.google.com/go/bigtable +module cloud.google.com/go/bigquery +module cloud.google.com/go/storage +module cloud.google.com/go/pubsublite +module cloud.google.com/go/firestore +module cloud.google.com/go/logging +module cloud.google.com/go/internal/gapicgen +module cloud.google.com/go/internal/godocfx +module cloud.google.com/go/internal/examples/fake +module cloud.google.com/go/internal/examples/mock +module cloud.google.com/go/datastore +``` + +The `cloud.google.com/go` is the repository root module. Each other module is +a submodule. + +So, if you need to release a change in `bigtable/bttest/inmem.go`, the closest +ancestor module is `cloud.google.com/go/bigtable` - so you should release a new +version of the `cloud.google.com/go/bigtable` submodule. + +If you need to release a change in `asset/apiv1/asset_client.go`, the closest +ancestor module is `cloud.google.com/go` - so you should release a new version +of the `cloud.google.com/go` repository root module. Note: releasing +`cloud.google.com/go` has no impact on any of the submodules, and vice-versa. +They are released entirely independently. + +## Test failures + +If there are any test failures in the Kokoro build, releases are blocked until +the failures have been resolved. + +## How to release + +### Automated Releases (`cloud.google.com/go` and submodules) + +We now use [release-please](https://github.com/googleapis/release-please) to +perform automated releases for `cloud.google.com/go` and all submodules. + +1. If there are changes that have not yet been released, a + [pull request](https://github.com/googleapis/google-cloud-go/pull/2971) will + be automatically opened by release-please + with a title like "chore: release X.Y.Z" (for the root module) or + "chore: release datastore X.Y.Z" (for the datastore submodule), where X.Y.Z + is the next version to be released. Find the desired pull request + [here](https://github.com/googleapis/google-cloud-go/pulls) +1. Check for failures in the + [continuous Kokoro build](http://go/google-cloud-go-continuous). If there are + any failures in the most recent build, address them before proceeding with + the release. (This applies even if the failures are in a different submodule + from the one being released.) +1. Review the release notes. These are automatically generated from the titles + of any merged commits since the previous release. If you would like to edit + them, this can be done by updating the changes in the release PR. +1. To cut a release, approve and merge the pull request. Doing so will + update the `CHANGES.md`, tag the merged commit with the appropriate version, + and draft a GitHub release which will copy the notes from `CHANGES.md`. + +### Manual Release (`cloud.google.com/go`) + +If for whatever reason the automated release process is not working as expected, +here is how to manually cut a release of `cloud.google.com/go`. + +1. Check for failures in the + [continuous Kokoro build](http://go/google-cloud-go-continuous). If there are + any failures in the most recent build, address them before proceeding with + the release. +1. Navigate to `google-cloud-go/` and switch to main. +1. `git pull` +1. Run `git tag -l | grep -v beta | grep -v alpha` to see all existing releases. + The current latest tag `$CV` is the largest tag. It should look something + like `vX.Y.Z` (note: ignore all `LIB/vX.Y.Z` tags - these are tags for a + specific library, not the module root). We'll call the current version `$CV` + and the new version `$NV`. +1. On main, run `git log $CV...` to list all the changes since the last + release. NOTE: You must manually visually parse out changes to submodules [1] + (the `git log` is going to show you things in submodules, which are not going + to be part of your release). +1. Edit `CHANGES.md` to include a summary of the changes. +1. In `internal/version/version.go`, update `const Repo` to today's date with + the format `YYYYMMDD`. +1. In `internal/version` run `go generate`. +1. Commit the changes, ignoring the generated `.go-r` file. Push to your fork, + and create a PR titled `chore: release $NV`. +1. Wait for the PR to be reviewed and merged. Once it's merged, and without + merging any other PRs in the meantime: + a. Switch to main. + b. `git pull` + c. Tag the repo with the next version: `git tag $NV`. + d. Push the tag to origin: + `git push origin $NV` +1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) + with the new release, copying the contents of `CHANGES.md`. + +### Manual Releases (submodules) + +If for whatever reason the automated release process is not working as expected, +here is how to manually cut a release of a submodule. + +(these instructions assume we're releasing `cloud.google.com/go/datastore` - adjust accordingly) + +1. Check for failures in the + [continuous Kokoro build](http://go/google-cloud-go-continuous). If there are + any failures in the most recent build, address them before proceeding with + the release. (This applies even if the failures are in a different submodule + from the one being released.) +1. Navigate to `google-cloud-go/` and switch to main. +1. `git pull` +1. Run `git tag -l | grep datastore | grep -v beta | grep -v alpha` to see all + existing releases. The current latest tag `$CV` is the largest tag. It + should look something like `datastore/vX.Y.Z`. We'll call the current version + `$CV` and the new version `$NV`. +1. On main, run `git log $CV.. -- datastore/` to list all the changes to the + submodule directory since the last release. +1. Edit `datastore/CHANGES.md` to include a summary of the changes. +1. In `internal/version` run `go generate`. +1. Commit the changes, ignoring the generated `.go-r` file. Push to your fork, + and create a PR titled `chore(datastore): release $NV`. +1. Wait for the PR to be reviewed and merged. Once it's merged, and without + merging any other PRs in the meantime: + a. Switch to main. + b. `git pull` + c. Tag the repo with the next version: `git tag $NV`. + d. Push the tag to origin: + `git push origin $NV` +1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) + with the new release, copying the contents of `datastore/CHANGES.md`. diff --git a/vendor/cloud.google.com/go/SECURITY.md b/vendor/cloud.google.com/go/SECURITY.md new file mode 100644 index 000000000..8b58ae9c0 --- /dev/null +++ b/vendor/cloud.google.com/go/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/vendor/cloud.google.com/go/compute/LICENSE b/vendor/cloud.google.com/go/compute/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/cloud.google.com/go/compute/internal/version.go b/vendor/cloud.google.com/go/compute/internal/version.go new file mode 100644 index 000000000..efedadbea --- /dev/null +++ b/vendor/cloud.google.com/go/compute/internal/version.go @@ -0,0 +1,18 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +// Version is the current tagged release of the library. +const Version = "1.13.0" diff --git a/vendor/cloud.google.com/go/compute/metadata/CHANGES.md b/vendor/cloud.google.com/go/compute/metadata/CHANGES.md new file mode 100644 index 000000000..8631b6d6d --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/CHANGES.md @@ -0,0 +1,5 @@ +# Changes + +## [0.1.0] (2022-10-26) + +Initial release of metadata being it's own module. diff --git a/vendor/cloud.google.com/go/compute/metadata/LICENSE b/vendor/cloud.google.com/go/compute/metadata/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/cloud.google.com/go/compute/metadata/README.md b/vendor/cloud.google.com/go/compute/metadata/README.md new file mode 100644 index 000000000..f940fb2c8 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/README.md @@ -0,0 +1,27 @@ +# Compute API + +[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/compute.svg)](https://pkg.go.dev/cloud.google.com/go/compute/metadata) + +This is a utility library for communicating with Google Cloud metadata service +on Google Cloud. + +## Install + +```bash +go get cloud.google.com/go/compute/metadata +``` + +## Go Version Support + +See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) +section in the root directory's README. + +## Contributing + +Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. diff --git a/vendor/cloud.google.com/go/compute/metadata/metadata.go b/vendor/cloud.google.com/go/compute/metadata/metadata.go new file mode 100644 index 000000000..50538b1d3 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/metadata.go @@ -0,0 +1,542 @@ +// Copyright 2014 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metadata provides access to Google Compute Engine (GCE) +// metadata and API service accounts. +// +// This package is a wrapper around the GCE metadata service, +// as documented at https://cloud.google.com/compute/docs/metadata/overview. +package metadata // import "cloud.google.com/go/compute/metadata" + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "runtime" + "strings" + "sync" + "time" +) + +const ( + // metadataIP is the documented metadata server IP address. + metadataIP = "169.254.169.254" + + // metadataHostEnv is the environment variable specifying the + // GCE metadata hostname. If empty, the default value of + // metadataIP ("169.254.169.254") is used instead. + // This is variable name is not defined by any spec, as far as + // I know; it was made up for the Go package. + metadataHostEnv = "GCE_METADATA_HOST" + + userAgent = "gcloud-golang/0.1" +) + +type cachedValue struct { + k string + trim bool + mu sync.Mutex + v string +} + +var ( + projID = &cachedValue{k: "project/project-id", trim: true} + projNum = &cachedValue{k: "project/numeric-project-id", trim: true} + instID = &cachedValue{k: "instance/id", trim: true} +) + +var defaultClient = &Client{hc: newDefaultHTTPClient()} + +func newDefaultHTTPClient() *http.Client { + return &http.Client{ + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 2 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + }, + Timeout: 5 * time.Second, + } +} + +// NotDefinedError is returned when requested metadata is not defined. +// +// The underlying string is the suffix after "/computeMetadata/v1/". +// +// This error is not returned if the value is defined to be the empty +// string. +type NotDefinedError string + +func (suffix NotDefinedError) Error() string { + return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix)) +} + +func (c *cachedValue) get(cl *Client) (v string, err error) { + defer c.mu.Unlock() + c.mu.Lock() + if c.v != "" { + return c.v, nil + } + if c.trim { + v, err = cl.getTrimmed(c.k) + } else { + v, err = cl.Get(c.k) + } + if err == nil { + c.v = v + } + return +} + +var ( + onGCEOnce sync.Once + onGCE bool +) + +// OnGCE reports whether this process is running on Google Compute Engine. +func OnGCE() bool { + onGCEOnce.Do(initOnGCE) + return onGCE +} + +func initOnGCE() { + onGCE = testOnGCE() +} + +func testOnGCE() bool { + // The user explicitly said they're on GCE, so trust them. + if os.Getenv(metadataHostEnv) != "" { + return true + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resc := make(chan bool, 2) + + // Try two strategies in parallel. + // See https://github.com/googleapis/google-cloud-go/issues/194 + go func() { + req, _ := http.NewRequest("GET", "http://"+metadataIP, nil) + req.Header.Set("User-Agent", userAgent) + res, err := newDefaultHTTPClient().Do(req.WithContext(ctx)) + if err != nil { + resc <- false + return + } + defer res.Body.Close() + resc <- res.Header.Get("Metadata-Flavor") == "Google" + }() + + go func() { + resolver := &net.Resolver{} + addrs, err := resolver.LookupHost(ctx, "metadata.google.internal") + if err != nil || len(addrs) == 0 { + resc <- false + return + } + resc <- strsContains(addrs, metadataIP) + }() + + tryHarder := systemInfoSuggestsGCE() + if tryHarder { + res := <-resc + if res { + // The first strategy succeeded, so let's use it. + return true + } + // Wait for either the DNS or metadata server probe to + // contradict the other one and say we are running on + // GCE. Give it a lot of time to do so, since the system + // info already suggests we're running on a GCE BIOS. + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case res = <-resc: + return res + case <-timer.C: + // Too slow. Who knows what this system is. + return false + } + } + + // There's no hint from the system info that we're running on + // GCE, so use the first probe's result as truth, whether it's + // true or false. The goal here is to optimize for speed for + // users who are NOT running on GCE. We can't assume that + // either a DNS lookup or an HTTP request to a blackholed IP + // address is fast. Worst case this should return when the + // metaClient's Transport.ResponseHeaderTimeout or + // Transport.Dial.Timeout fires (in two seconds). + return <-resc +} + +// systemInfoSuggestsGCE reports whether the local system (without +// doing network requests) suggests that we're running on GCE. If this +// returns true, testOnGCE tries a bit harder to reach its metadata +// server. +func systemInfoSuggestsGCE() bool { + if runtime.GOOS != "linux" { + // We don't have any non-Linux clues available, at least yet. + return false + } + slurp, _ := ioutil.ReadFile("/sys/class/dmi/id/product_name") + name := strings.TrimSpace(string(slurp)) + return name == "Google" || name == "Google Compute Engine" +} + +// Subscribe calls Client.Subscribe on the default client. +func Subscribe(suffix string, fn func(v string, ok bool) error) error { + return defaultClient.Subscribe(suffix, fn) +} + +// Get calls Client.Get on the default client. +func Get(suffix string) (string, error) { return defaultClient.Get(suffix) } + +// ProjectID returns the current instance's project ID string. +func ProjectID() (string, error) { return defaultClient.ProjectID() } + +// NumericProjectID returns the current instance's numeric project ID. +func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() } + +// InternalIP returns the instance's primary internal IP address. +func InternalIP() (string, error) { return defaultClient.InternalIP() } + +// ExternalIP returns the instance's primary external (public) IP address. +func ExternalIP() (string, error) { return defaultClient.ExternalIP() } + +// Email calls Client.Email on the default client. +func Email(serviceAccount string) (string, error) { return defaultClient.Email(serviceAccount) } + +// Hostname returns the instance's hostname. This will be of the form +// ".c..internal". +func Hostname() (string, error) { return defaultClient.Hostname() } + +// InstanceTags returns the list of user-defined instance tags, +// assigned when initially creating a GCE instance. +func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() } + +// InstanceID returns the current VM's numeric instance ID. +func InstanceID() (string, error) { return defaultClient.InstanceID() } + +// InstanceName returns the current VM's instance ID string. +func InstanceName() (string, error) { return defaultClient.InstanceName() } + +// Zone returns the current VM's zone, such as "us-central1-b". +func Zone() (string, error) { return defaultClient.Zone() } + +// InstanceAttributes calls Client.InstanceAttributes on the default client. +func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() } + +// ProjectAttributes calls Client.ProjectAttributes on the default client. +func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() } + +// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client. +func InstanceAttributeValue(attr string) (string, error) { + return defaultClient.InstanceAttributeValue(attr) +} + +// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client. +func ProjectAttributeValue(attr string) (string, error) { + return defaultClient.ProjectAttributeValue(attr) +} + +// Scopes calls Client.Scopes on the default client. +func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) } + +func strsContains(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } + } + return false +} + +// A Client provides metadata. +type Client struct { + hc *http.Client +} + +// NewClient returns a Client that can be used to fetch metadata. +// Returns the client that uses the specified http.Client for HTTP requests. +// If nil is specified, returns the default client. +func NewClient(c *http.Client) *Client { + if c == nil { + return defaultClient + } + + return &Client{hc: c} +} + +// getETag returns a value from the metadata service as well as the associated ETag. +// This func is otherwise equivalent to Get. +func (c *Client) getETag(suffix string) (value, etag string, err error) { + ctx := context.TODO() + // Using a fixed IP makes it very difficult to spoof the metadata service in + // a container, which is an important use-case for local testing of cloud + // deployments. To enable spoofing of the metadata service, the environment + // variable GCE_METADATA_HOST is first inspected to decide where metadata + // requests shall go. + host := os.Getenv(metadataHostEnv) + if host == "" { + // Using 169.254.169.254 instead of "metadata" here because Go + // binaries built with the "netgo" tag and without cgo won't + // know the search suffix for "metadata" is + // ".google.internal", and this IP address is documented as + // being stable anyway. + host = metadataIP + } + suffix = strings.TrimLeft(suffix, "/") + u := "http://" + host + "/computeMetadata/v1/" + suffix + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return "", "", err + } + req.Header.Set("Metadata-Flavor", "Google") + req.Header.Set("User-Agent", userAgent) + var res *http.Response + var reqErr error + retryer := newRetryer() + for { + res, reqErr = c.hc.Do(req) + var code int + if res != nil { + code = res.StatusCode + } + if delay, shouldRetry := retryer.Retry(code, reqErr); shouldRetry { + if err := sleep(ctx, delay); err != nil { + return "", "", err + } + continue + } + break + } + if reqErr != nil { + return "", "", reqErr + } + defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return "", "", NotDefinedError(suffix) + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", err + } + if res.StatusCode != 200 { + return "", "", &Error{Code: res.StatusCode, Message: string(all)} + } + return string(all), res.Header.Get("Etag"), nil +} + +// Get returns a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// +// If the GCE_METADATA_HOST environment variable is not defined, a default of +// 169.254.169.254 will be used instead. +// +// If the requested metadata is not defined, the returned error will +// be of type NotDefinedError. +func (c *Client) Get(suffix string) (string, error) { + val, _, err := c.getETag(suffix) + return val, err +} + +func (c *Client) getTrimmed(suffix string) (s string, err error) { + s, err = c.Get(suffix) + s = strings.TrimSpace(s) + return +} + +func (c *Client) lines(suffix string) ([]string, error) { + j, err := c.Get(suffix) + if err != nil { + return nil, err + } + s := strings.Split(strings.TrimSpace(j), "\n") + for i := range s { + s[i] = strings.TrimSpace(s[i]) + } + return s, nil +} + +// ProjectID returns the current instance's project ID string. +func (c *Client) ProjectID() (string, error) { return projID.get(c) } + +// NumericProjectID returns the current instance's numeric project ID. +func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) } + +// InstanceID returns the current VM's numeric instance ID. +func (c *Client) InstanceID() (string, error) { return instID.get(c) } + +// InternalIP returns the instance's primary internal IP address. +func (c *Client) InternalIP() (string, error) { + return c.getTrimmed("instance/network-interfaces/0/ip") +} + +// Email returns the email address associated with the service account. +// The account may be empty or the string "default" to use the instance's +// main account. +func (c *Client) Email(serviceAccount string) (string, error) { + if serviceAccount == "" { + serviceAccount = "default" + } + return c.getTrimmed("instance/service-accounts/" + serviceAccount + "/email") +} + +// ExternalIP returns the instance's primary external (public) IP address. +func (c *Client) ExternalIP() (string, error) { + return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") +} + +// Hostname returns the instance's hostname. This will be of the form +// ".c..internal". +func (c *Client) Hostname() (string, error) { + return c.getTrimmed("instance/hostname") +} + +// InstanceTags returns the list of user-defined instance tags, +// assigned when initially creating a GCE instance. +func (c *Client) InstanceTags() ([]string, error) { + var s []string + j, err := c.Get("instance/tags") + if err != nil { + return nil, err + } + if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil { + return nil, err + } + return s, nil +} + +// InstanceName returns the current VM's instance ID string. +func (c *Client) InstanceName() (string, error) { + return c.getTrimmed("instance/name") +} + +// Zone returns the current VM's zone, such as "us-central1-b". +func (c *Client) Zone() (string, error) { + zone, err := c.getTrimmed("instance/zone") + // zone is of the form "projects//zones/". + if err != nil { + return "", err + } + return zone[strings.LastIndex(zone, "/")+1:], nil +} + +// InstanceAttributes returns the list of user-defined attributes, +// assigned when initially creating a GCE VM instance. The value of an +// attribute can be obtained with InstanceAttributeValue. +func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") } + +// ProjectAttributes returns the list of user-defined attributes +// applying to the project as a whole, not just this VM. The value of +// an attribute can be obtained with ProjectAttributeValue. +func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") } + +// InstanceAttributeValue returns the value of the provided VM +// instance attribute. +// +// If the requested attribute is not defined, the returned error will +// be of type NotDefinedError. +// +// InstanceAttributeValue may return ("", nil) if the attribute was +// defined to be the empty string. +func (c *Client) InstanceAttributeValue(attr string) (string, error) { + return c.Get("instance/attributes/" + attr) +} + +// ProjectAttributeValue returns the value of the provided +// project attribute. +// +// If the requested attribute is not defined, the returned error will +// be of type NotDefinedError. +// +// ProjectAttributeValue may return ("", nil) if the attribute was +// defined to be the empty string. +func (c *Client) ProjectAttributeValue(attr string) (string, error) { + return c.Get("project/attributes/" + attr) +} + +// Scopes returns the service account scopes for the given account. +// The account may be empty or the string "default" to use the instance's +// main account. +func (c *Client) Scopes(serviceAccount string) ([]string, error) { + if serviceAccount == "" { + serviceAccount = "default" + } + return c.lines("instance/service-accounts/" + serviceAccount + "/scopes") +} + +// Subscribe subscribes to a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// The suffix may contain query parameters. +// +// Subscribe calls fn with the latest metadata value indicated by the provided +// suffix. If the metadata value is deleted, fn is called with the empty string +// and ok false. Subscribe blocks until fn returns a non-nil error or the value +// is deleted. Subscribe returns the error value returned from the last call to +// fn, which may be nil when ok == false. +func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error { + const failedSubscribeSleep = time.Second * 5 + + // First check to see if the metadata value exists at all. + val, lastETag, err := c.getETag(suffix) + if err != nil { + return err + } + + if err := fn(val, true); err != nil { + return err + } + + ok := true + if strings.ContainsRune(suffix, '?') { + suffix += "&wait_for_change=true&last_etag=" + } else { + suffix += "?wait_for_change=true&last_etag=" + } + for { + val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag)) + if err != nil { + if _, deleted := err.(NotDefinedError); !deleted { + time.Sleep(failedSubscribeSleep) + continue // Retry on other errors. + } + ok = false + } + lastETag = etag + + if err := fn(val, ok); err != nil || !ok { + return err + } + } +} + +// Error contains an error response from the server. +type Error struct { + // Code is the HTTP response status code. + Code int + // Message is the server response message. + Message string +} + +func (e *Error) Error() string { + return fmt.Sprintf("compute: Received %d `%s`", e.Code, e.Message) +} diff --git a/vendor/cloud.google.com/go/compute/metadata/retry.go b/vendor/cloud.google.com/go/compute/metadata/retry.go new file mode 100644 index 000000000..0f18f3cda --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/retry.go @@ -0,0 +1,114 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "context" + "io" + "math/rand" + "net/http" + "time" +) + +const ( + maxRetryAttempts = 5 +) + +var ( + syscallRetryable = func(err error) bool { return false } +) + +// defaultBackoff is basically equivalent to gax.Backoff without the need for +// the dependency. +type defaultBackoff struct { + max time.Duration + mul float64 + cur time.Duration +} + +func (b *defaultBackoff) Pause() time.Duration { + d := time.Duration(1 + rand.Int63n(int64(b.cur))) + b.cur = time.Duration(float64(b.cur) * b.mul) + if b.cur > b.max { + b.cur = b.max + } + return d +} + +// sleep is the equivalent of gax.Sleep without the need for the dependency. +func sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + select { + case <-ctx.Done(): + t.Stop() + return ctx.Err() + case <-t.C: + return nil + } +} + +func newRetryer() *metadataRetryer { + return &metadataRetryer{bo: &defaultBackoff{ + cur: 100 * time.Millisecond, + max: 30 * time.Second, + mul: 2, + }} +} + +type backoff interface { + Pause() time.Duration +} + +type metadataRetryer struct { + bo backoff + attempts int +} + +func (r *metadataRetryer) Retry(status int, err error) (time.Duration, bool) { + if status == http.StatusOK { + return 0, false + } + retryOk := shouldRetry(status, err) + if !retryOk { + return 0, false + } + if r.attempts == maxRetryAttempts { + return 0, false + } + r.attempts++ + return r.bo.Pause(), true +} + +func shouldRetry(status int, err error) bool { + if 500 <= status && status <= 599 { + return true + } + if err == io.ErrUnexpectedEOF { + return true + } + // Transient network errors should be retried. + if syscallRetryable(err) { + return true + } + if err, ok := err.(interface{ Temporary() bool }); ok { + if err.Temporary() { + return true + } + } + if err, ok := err.(interface{ Unwrap() error }); ok { + return shouldRetry(status, err.Unwrap()) + } + return false +} diff --git a/vendor/cloud.google.com/go/compute/metadata/retry_linux.go b/vendor/cloud.google.com/go/compute/metadata/retry_linux.go new file mode 100644 index 000000000..bb412f891 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/retry_linux.go @@ -0,0 +1,26 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package metadata + +import "syscall" + +func init() { + // Initialize syscallRetryable to return true on transient socket-level + // errors. These errors are specific to Linux. + syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED } +} diff --git a/vendor/cloud.google.com/go/compute/metadata/tidyfix.go b/vendor/cloud.google.com/go/compute/metadata/tidyfix.go new file mode 100644 index 000000000..4cef48500 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/tidyfix.go @@ -0,0 +1,23 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file, and the {{.RootMod}} import, won't actually become part of +// the resultant binary. +//go:build modhack +// +build modhack + +package metadata + +// Necessary for safely adding multi-module repo. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository +import _ "cloud.google.com/go/compute/internal" diff --git a/vendor/cloud.google.com/go/doc.go b/vendor/cloud.google.com/go/doc.go new file mode 100644 index 000000000..833878ec8 --- /dev/null +++ b/vendor/cloud.google.com/go/doc.go @@ -0,0 +1,248 @@ +// Copyright 2014 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package cloud is the root of the packages used to access Google Cloud +Services. See https://godoc.org/cloud.google.com/go for a full list +of sub-packages. + +# Client Options + +All clients in sub-packages are configurable via client options. These options are +described here: https://godoc.org/google.golang.org/api/option. + +## Endpoint Override + +Endpoint configuration is used to specify the URL to which requests are +sent. It is used for services that support or require regional endpoints, as well +as for other use cases such as [testing against fake +servers](https://github.com/googleapis/google-cloud-go/blob/main/testing.md#testing-grpc-services-using-fakes). + +For example, the Vertex AI service recommends that you configure the endpoint to the +location with the features you want that is closest to your physical location or the +location of your users. There is no global endpoint for Vertex AI. See +[Vertex AI - Locations](https://cloud.google.com/vertex-ai/docs/general/locations) +for more details. The following example demonstrates configuring a Vertex AI client +with a regional endpoint: + + ctx := context.Background() + endpoint := "us-central1-aiplatform.googleapis.com:443" + client, err := aiplatform.NewDatasetClient(ctx, option.WithEndpoint(endpoint)) + +# Authentication and Authorization + +All the clients in sub-packages support authentication via Google Application Default +Credentials (see https://cloud.google.com/docs/authentication/production), or +by providing a JSON key file for a Service Account. See examples below. + +Google Application Default Credentials (ADC) is the recommended way to authorize +and authenticate clients. For information on how to create and obtain +Application Default Credentials, see +https://cloud.google.com/docs/authentication/production. Here is an example +of a client using ADC to authenticate: + + client, err := secretmanager.NewClient(context.Background()) + if err != nil { + // TODO: handle error. + } + _ = client // Use the client. + +You can use a file with credentials to authenticate and authorize, such as a JSON +key file associated with a Google service account. Service Account keys can be +created and downloaded from +https://console.cloud.google.com/iam-admin/serviceaccounts. This example uses +the Secret Manger client, but the same steps apply to the other client libraries +underneath this package. Example: + + client, err := secretmanager.NewClient(context.Background(), + option.WithCredentialsFile("/path/to/service-account-key.json")) + if err != nil { + // TODO: handle error. + } + _ = client // Use the client. + +In some cases (for instance, you don't want to store secrets on disk), you can +create credentials from in-memory JSON and use the WithCredentials option. +The google package in this example is at golang.org/x/oauth2/google. +This example uses the Secret Manager client, but the same steps apply to +the other client libraries underneath this package. Note that scopes can be +found at https://developers.google.com/identity/protocols/oauth2/scopes, and +are also provided in all auto-generated libraries: for example, +cloud.google.com/go/secretmanager/apiv1 provides DefaultAuthScopes. Example: + + ctx := context.Background() + creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), secretmanager.DefaultAuthScopes()...) + if err != nil { + // TODO: handle error. + } + client, err := secretmanager.NewClient(ctx, option.WithCredentials(creds)) + if err != nil { + // TODO: handle error. + } + _ = client // Use the client. + +# Timeouts and Cancellation + +By default, non-streaming methods, like Create or Get, will have a default deadline applied to the +context provided at call time, unless a context deadline is already set. Streaming +methods have no default deadline and will run indefinitely. To set timeouts or +arrange for cancellation, use contexts. Transient +errors will be retried when correctness allows. + +Here is an example of how to set a timeout for an RPC, use context.WithTimeout: + + ctx := context.Background() + // Do not set a timeout on the context passed to NewClient: dialing happens + // asynchronously, and the context is used to refresh credentials in the + // background. + client, err := secretmanager.NewClient(ctx) + if err != nil { + // TODO: handle error. + } + // Time out if it takes more than 10 seconds to create a dataset. + tctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() // Always call cancel. + + req := &secretmanagerpb.DeleteSecretRequest{Name: "projects/project-id/secrets/name"} + if err := client.DeleteSecret(tctx, req); err != nil { + // TODO: handle error. + } + +Here is an example of how to arrange for an RPC to be canceled, use context.WithCancel: + + ctx := context.Background() + // Do not cancel the context passed to NewClient: dialing happens asynchronously, + // and the context is used to refresh credentials in the background. + client, err := secretmanager.NewClient(ctx) + if err != nil { + // TODO: handle error. + } + cctx, cancel := context.WithCancel(ctx) + defer cancel() // Always call cancel. + + // TODO: Make the cancel function available to whatever might want to cancel the + // call--perhaps a GUI button. + req := &secretmanagerpb.DeleteSecretRequest{Name: "projects/proj/secrets/name"} + if err := client.DeleteSecret(cctx, req); err != nil { + // TODO: handle error. + } + +To opt out of default deadlines, set the temporary environment variable +GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE to "true" prior to client +creation. This affects all Google Cloud Go client libraries. This opt-out +mechanism will be removed in a future release. File an issue at +https://github.com/googleapis/google-cloud-go if the default deadlines +cannot work for you. + +Do not attempt to control the initial connection (dialing) of a service by setting a +timeout on the context passed to NewClient. Dialing is non-blocking, so timeouts +would be ineffective and would only interfere with credential refreshing, which uses +the same context. + +# Connection Pooling + +Connection pooling differs in clients based on their transport. Cloud +clients either rely on HTTP or gRPC transports to communicate +with Google Cloud. + +Cloud clients that use HTTP (bigquery, compute, storage, and translate) rely on the +underlying HTTP transport to cache connections for later re-use. These are cached to +the default http.MaxIdleConns and http.MaxIdleConnsPerHost settings in +http.DefaultTransport. + +For gRPC clients (all others in this repo), connection pooling is configurable. Users +of cloud client libraries may specify option.WithGRPCConnectionPool(n) as a client +option to NewClient calls. This configures the underlying gRPC connections to be +pooled and addressed in a round robin fashion. + +# Using the Libraries with Docker + +Minimal docker images like Alpine lack CA certificates. This causes RPCs to appear to +hang, because gRPC retries indefinitely. See https://github.com/googleapis/google-cloud-go/issues/928 +for more information. + +# Debugging + +To see gRPC logs, set the environment variable GRPC_GO_LOG_SEVERITY_LEVEL. See +https://godoc.org/google.golang.org/grpc/grpclog for more information. + +For HTTP logging, set the GODEBUG environment variable to "http2debug=1" or "http2debug=2". + +# Inspecting errors + +Most of the errors returned by the generated clients are wrapped in an +[github.com/googleapis/gax-go/v2/apierror.APIError] and can be further unwrapped +into a [google.golang.org/grpc/status.Status] or +[google.golang.org/api/googleapi.Error] depending +on the transport used to make the call (gRPC or REST). Converting your errors to +these types can be a useful way to get more information about what went wrong +while debugging. + +[github.com/googleapis/gax-go/v2/apierror.APIError] gives access to specific +details in the error. The transport-specific errors can still be unwrapped using +the [github.com/googleapis/gax-go/v2/apierror.APIError]. + + if err != nil { + var ae *apierror.APIError + if errors.As(err, &ae) { + log.Println(ae.Reason()) + log.Println(ae.Details().Help.GetLinks()) + } + } + +If the gRPC transport was used, the [google.golang.org/grpc/status.Status] can +still be parsed using the [google.golang.org/grpc/status.FromError] function. + + if err != nil { + if s, ok := status.FromError(err); ok { + log.Println(s.Message()) + for _, d := range s.Proto().Details { + log.Println(d) + } + } + } + +If the REST transport was used, the [google.golang.org/api/googleapi.Error] can +be parsed in a similar way, allowing access to details such as the HTTP response +code. + + if err != nil { + var gerr *googleapi.Error + if errors.As(err, &gerr) { + log.Println(gerr.Message) + } + } + +# Client Stability + +Clients in this repository are considered alpha or beta unless otherwise +marked as stable in the README.md. Semver is not used to communicate stability +of clients. + +Alpha and beta clients may change or go away without notice. + +Clients marked stable will maintain compatibility with future versions for as +long as we can reasonably sustain. Incompatible changes might be made in some +situations, including: + +- Security bugs may prompt backwards-incompatible changes. + +- Situations in which components are no longer feasible to maintain without +making breaking changes, including removal. + +- Parts of the client surface may be outright unstable and subject to change. +These parts of the surface will be labeled with the note, "It is EXPERIMENTAL +and subject to change or removal without notice." +*/ +package cloud // import "cloud.google.com/go" diff --git a/vendor/cloud.google.com/go/iam/CHANGES.md b/vendor/cloud.google.com/go/iam/CHANGES.md new file mode 100644 index 000000000..ced217827 --- /dev/null +++ b/vendor/cloud.google.com/go/iam/CHANGES.md @@ -0,0 +1,62 @@ +# Changes + +## [0.8.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.7.0...iam/v0.8.0) (2022-12-05) + + +### Features + +* **iam:** Start generating and refresh some libraries ([#7089](https://github.com/googleapis/google-cloud-go/issues/7089)) ([a9045ff](https://github.com/googleapis/google-cloud-go/commit/a9045ff191a711089c37f1d94a63522d9939ce38)) + +## [0.7.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.6.0...iam/v0.7.0) (2022-11-03) + + +### Features + +* **iam:** rewrite signatures in terms of new location ([3c4b2b3](https://github.com/googleapis/google-cloud-go/commit/3c4b2b34565795537aac1661e6af2442437e34ad)) + +## [0.6.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.5.0...iam/v0.6.0) (2022-10-25) + + +### Features + +* **iam:** start generating stubs dir ([de2d180](https://github.com/googleapis/google-cloud-go/commit/de2d18066dc613b72f6f8db93ca60146dabcfdcc)) + +## [0.5.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.4.0...iam/v0.5.0) (2022-09-28) + + +### Features + +* **iam:** remove ListApplicablePolicies ([52dddd1](https://github.com/googleapis/google-cloud-go/commit/52dddd1ed89fbe77e1859311c3b993a77a82bfc7)) + +## [0.4.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.3.0...iam/v0.4.0) (2022-09-06) + + +### Features + +* **iam:** start generating apiv2 ([#6605](https://github.com/googleapis/google-cloud-go/issues/6605)) ([a6004e7](https://github.com/googleapis/google-cloud-go/commit/a6004e762f782869cd85688937475744f7b17e50)) + +## [0.3.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.2.0...iam/v0.3.0) (2022-02-23) + + +### Features + +* **iam:** set versionClient to module version ([55f0d92](https://github.com/googleapis/google-cloud-go/commit/55f0d92bf112f14b024b4ab0076c9875a17423c9)) + +## [0.2.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.1.1...iam/v0.2.0) (2022-02-14) + + +### Features + +* **iam:** add file for tracking version ([17b36ea](https://github.com/googleapis/google-cloud-go/commit/17b36ead42a96b1a01105122074e65164357519e)) + +### [0.1.1](https://www.github.com/googleapis/google-cloud-go/compare/iam/v0.1.0...iam/v0.1.1) (2022-01-14) + + +### Bug Fixes + +* **iam:** run formatter ([#5277](https://www.github.com/googleapis/google-cloud-go/issues/5277)) ([8682e4e](https://www.github.com/googleapis/google-cloud-go/commit/8682e4ed57a4428a659fbc225f56c91767e2a4a9)) + +## v0.1.0 + +This is the first tag to carve out iam as its own module. See +[Add a module to a multi-module repository](https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository). diff --git a/vendor/cloud.google.com/go/iam/LICENSE b/vendor/cloud.google.com/go/iam/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/cloud.google.com/go/iam/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/cloud.google.com/go/iam/README.md b/vendor/cloud.google.com/go/iam/README.md new file mode 100644 index 000000000..0072cc9e2 --- /dev/null +++ b/vendor/cloud.google.com/go/iam/README.md @@ -0,0 +1,40 @@ +# IAM API + +[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/iam.svg)](https://pkg.go.dev/cloud.google.com/go/iam) + +Go Client Library for IAM API. + +## Install + +```bash +go get cloud.google.com/go/iam +``` + +## Stability + +The stability of this module is indicated by SemVer. + +However, a `v1+` module may have breaking changes in two scenarios: + +* Packages with `alpha` or `beta` in the import path +* The GoDoc has an explicit stability disclaimer (for example, for an experimental feature). + +## Go Version Support + +See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) +section in the root directory's README. + +## Authorization + +See the [Authorization](https://github.com/googleapis/google-cloud-go#authorization) +section in the root directory's README. + +## Contributing + +Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. diff --git a/vendor/cloud.google.com/go/iam/iam.go b/vendor/cloud.google.com/go/iam/iam.go new file mode 100644 index 000000000..0a06ea2e8 --- /dev/null +++ b/vendor/cloud.google.com/go/iam/iam.go @@ -0,0 +1,387 @@ +// Copyright 2016 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package iam supports the resource-specific operations of Google Cloud +// IAM (Identity and Access Management) for the Google Cloud Libraries. +// See https://cloud.google.com/iam for more about IAM. +// +// Users of the Google Cloud Libraries will typically not use this package +// directly. Instead they will begin with some resource that supports IAM, like +// a pubsub topic, and call its IAM method to get a Handle for that resource. +package iam + +import ( + "context" + "fmt" + "time" + + gax "github.com/googleapis/gax-go/v2" + pb "google.golang.org/genproto/googleapis/iam/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" +) + +// client abstracts the IAMPolicy API to allow multiple implementations. +type client interface { + Get(ctx context.Context, resource string) (*pb.Policy, error) + Set(ctx context.Context, resource string, p *pb.Policy) error + Test(ctx context.Context, resource string, perms []string) ([]string, error) + GetWithVersion(ctx context.Context, resource string, requestedPolicyVersion int32) (*pb.Policy, error) +} + +// grpcClient implements client for the standard gRPC-based IAMPolicy service. +type grpcClient struct { + c pb.IAMPolicyClient +} + +var withRetry = gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 100 * time.Millisecond, + Max: 60 * time.Second, + Multiplier: 1.3, + }) +}) + +func (g *grpcClient) Get(ctx context.Context, resource string) (*pb.Policy, error) { + return g.GetWithVersion(ctx, resource, 1) +} + +func (g *grpcClient) GetWithVersion(ctx context.Context, resource string, requestedPolicyVersion int32) (*pb.Policy, error) { + var proto *pb.Policy + md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "resource", resource)) + ctx = insertMetadata(ctx, md) + + err := gax.Invoke(ctx, func(ctx context.Context, _ gax.CallSettings) error { + var err error + proto, err = g.c.GetIamPolicy(ctx, &pb.GetIamPolicyRequest{ + Resource: resource, + Options: &pb.GetPolicyOptions{ + RequestedPolicyVersion: requestedPolicyVersion, + }, + }) + return err + }, withRetry) + if err != nil { + return nil, err + } + return proto, nil +} + +func (g *grpcClient) Set(ctx context.Context, resource string, p *pb.Policy) error { + md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "resource", resource)) + ctx = insertMetadata(ctx, md) + + return gax.Invoke(ctx, func(ctx context.Context, _ gax.CallSettings) error { + _, err := g.c.SetIamPolicy(ctx, &pb.SetIamPolicyRequest{ + Resource: resource, + Policy: p, + }) + return err + }, withRetry) +} + +func (g *grpcClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) { + var res *pb.TestIamPermissionsResponse + md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "resource", resource)) + ctx = insertMetadata(ctx, md) + + err := gax.Invoke(ctx, func(ctx context.Context, _ gax.CallSettings) error { + var err error + res, err = g.c.TestIamPermissions(ctx, &pb.TestIamPermissionsRequest{ + Resource: resource, + Permissions: perms, + }) + return err + }, withRetry) + if err != nil { + return nil, err + } + return res.Permissions, nil +} + +// A Handle provides IAM operations for a resource. +type Handle struct { + c client + resource string +} + +// A Handle3 provides IAM operations for a resource. It is similar to a Handle, but provides access to newer IAM features (e.g., conditions). +type Handle3 struct { + c client + resource string + version int32 +} + +// InternalNewHandle is for use by the Google Cloud Libraries only. +// +// InternalNewHandle returns a Handle for resource. +// The conn parameter refers to a server that must support the IAMPolicy service. +func InternalNewHandle(conn grpc.ClientConnInterface, resource string) *Handle { + return InternalNewHandleGRPCClient(pb.NewIAMPolicyClient(conn), resource) +} + +// InternalNewHandleGRPCClient is for use by the Google Cloud Libraries only. +// +// InternalNewHandleClient returns a Handle for resource using the given +// grpc service that implements IAM as a mixin +func InternalNewHandleGRPCClient(c pb.IAMPolicyClient, resource string) *Handle { + return InternalNewHandleClient(&grpcClient{c: c}, resource) +} + +// InternalNewHandleClient is for use by the Google Cloud Libraries only. +// +// InternalNewHandleClient returns a Handle for resource using the given +// client implementation. +func InternalNewHandleClient(c client, resource string) *Handle { + return &Handle{ + c: c, + resource: resource, + } +} + +// V3 returns a Handle3, which is like Handle except it sets +// requestedPolicyVersion to 3 when retrieving a policy and policy.version to 3 +// when storing a policy. +func (h *Handle) V3() *Handle3 { + return &Handle3{ + c: h.c, + resource: h.resource, + version: 3, + } +} + +// Policy retrieves the IAM policy for the resource. +func (h *Handle) Policy(ctx context.Context) (*Policy, error) { + proto, err := h.c.Get(ctx, h.resource) + if err != nil { + return nil, err + } + return &Policy{InternalProto: proto}, nil +} + +// SetPolicy replaces the resource's current policy with the supplied Policy. +// +// If policy was created from a prior call to Get, then the modification will +// only succeed if the policy has not changed since the Get. +func (h *Handle) SetPolicy(ctx context.Context, policy *Policy) error { + return h.c.Set(ctx, h.resource, policy.InternalProto) +} + +// TestPermissions returns the subset of permissions that the caller has on the resource. +func (h *Handle) TestPermissions(ctx context.Context, permissions []string) ([]string, error) { + return h.c.Test(ctx, h.resource, permissions) +} + +// A RoleName is a name representing a collection of permissions. +type RoleName string + +// Common role names. +const ( + Owner RoleName = "roles/owner" + Editor RoleName = "roles/editor" + Viewer RoleName = "roles/viewer" +) + +const ( + // AllUsers is a special member that denotes all users, even unauthenticated ones. + AllUsers = "allUsers" + + // AllAuthenticatedUsers is a special member that denotes all authenticated users. + AllAuthenticatedUsers = "allAuthenticatedUsers" +) + +// A Policy is a list of Bindings representing roles +// granted to members. +// +// The zero Policy is a valid policy with no bindings. +type Policy struct { + // TODO(jba): when type aliases are available, put Policy into an internal package + // and provide an exported alias here. + + // This field is exported for use by the Google Cloud Libraries only. + // It may become unexported in a future release. + InternalProto *pb.Policy +} + +// Members returns the list of members with the supplied role. +// The return value should not be modified. Use Add and Remove +// to modify the members of a role. +func (p *Policy) Members(r RoleName) []string { + b := p.binding(r) + if b == nil { + return nil + } + return b.Members +} + +// HasRole reports whether member has role r. +func (p *Policy) HasRole(member string, r RoleName) bool { + return memberIndex(member, p.binding(r)) >= 0 +} + +// Add adds member member to role r if it is not already present. +// A new binding is created if there is no binding for the role. +func (p *Policy) Add(member string, r RoleName) { + b := p.binding(r) + if b == nil { + if p.InternalProto == nil { + p.InternalProto = &pb.Policy{} + } + p.InternalProto.Bindings = append(p.InternalProto.Bindings, &pb.Binding{ + Role: string(r), + Members: []string{member}, + }) + return + } + if memberIndex(member, b) < 0 { + b.Members = append(b.Members, member) + return + } +} + +// Remove removes member from role r if it is present. +func (p *Policy) Remove(member string, r RoleName) { + bi := p.bindingIndex(r) + if bi < 0 { + return + } + bindings := p.InternalProto.Bindings + b := bindings[bi] + mi := memberIndex(member, b) + if mi < 0 { + return + } + // Order doesn't matter for bindings or members, so to remove, move the last item + // into the removed spot and shrink the slice. + if len(b.Members) == 1 { + // Remove binding. + last := len(bindings) - 1 + bindings[bi] = bindings[last] + bindings[last] = nil + p.InternalProto.Bindings = bindings[:last] + return + } + // Remove member. + // TODO(jba): worry about multiple copies of m? + last := len(b.Members) - 1 + b.Members[mi] = b.Members[last] + b.Members[last] = "" + b.Members = b.Members[:last] +} + +// Roles returns the names of all the roles that appear in the Policy. +func (p *Policy) Roles() []RoleName { + if p.InternalProto == nil { + return nil + } + var rns []RoleName + for _, b := range p.InternalProto.Bindings { + rns = append(rns, RoleName(b.Role)) + } + return rns +} + +// binding returns the Binding for the suppied role, or nil if there isn't one. +func (p *Policy) binding(r RoleName) *pb.Binding { + i := p.bindingIndex(r) + if i < 0 { + return nil + } + return p.InternalProto.Bindings[i] +} + +func (p *Policy) bindingIndex(r RoleName) int { + if p.InternalProto == nil { + return -1 + } + for i, b := range p.InternalProto.Bindings { + if b.Role == string(r) { + return i + } + } + return -1 +} + +// memberIndex returns the index of m in b's Members, or -1 if not found. +func memberIndex(m string, b *pb.Binding) int { + if b == nil { + return -1 + } + for i, mm := range b.Members { + if mm == m { + return i + } + } + return -1 +} + +// insertMetadata inserts metadata into the given context +func insertMetadata(ctx context.Context, mds ...metadata.MD) context.Context { + out, _ := metadata.FromOutgoingContext(ctx) + out = out.Copy() + for _, md := range mds { + for k, v := range md { + out[k] = append(out[k], v...) + } + } + return metadata.NewOutgoingContext(ctx, out) +} + +// A Policy3 is a list of Bindings representing roles granted to members. +// +// The zero Policy3 is a valid policy with no bindings. +// +// It is similar to a Policy, except a Policy3 provides direct access to the +// list of Bindings. +// +// The policy version is always set to 3. +type Policy3 struct { + etag []byte + Bindings []*pb.Binding +} + +// Policy retrieves the IAM policy for the resource. +// +// requestedPolicyVersion is always set to 3. +func (h *Handle3) Policy(ctx context.Context) (*Policy3, error) { + proto, err := h.c.GetWithVersion(ctx, h.resource, h.version) + if err != nil { + return nil, err + } + return &Policy3{ + Bindings: proto.Bindings, + etag: proto.Etag, + }, nil +} + +// SetPolicy replaces the resource's current policy with the supplied Policy. +// +// If policy was created from a prior call to Get, then the modification will +// only succeed if the policy has not changed since the Get. +func (h *Handle3) SetPolicy(ctx context.Context, policy *Policy3) error { + return h.c.Set(ctx, h.resource, &pb.Policy{ + Bindings: policy.Bindings, + Etag: policy.etag, + Version: h.version, + }) +} + +// TestPermissions returns the subset of permissions that the caller has on the resource. +func (h *Handle3) TestPermissions(ctx context.Context, permissions []string) ([]string, error) { + return h.c.Test(ctx, h.resource, permissions) +} diff --git a/vendor/cloud.google.com/go/internal/.repo-metadata-full.json b/vendor/cloud.google.com/go/internal/.repo-metadata-full.json new file mode 100644 index 000000000..39bc0be8c --- /dev/null +++ b/vendor/cloud.google.com/go/internal/.repo-metadata-full.json @@ -0,0 +1,1946 @@ +{ + "cloud.google.com/go/accessapproval/apiv1": { + "distribution_name": "cloud.google.com/go/accessapproval/apiv1", + "description": "Access Approval API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/accessapproval/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/accesscontextmanager/apiv1": { + "distribution_name": "cloud.google.com/go/accesscontextmanager/apiv1", + "description": "Access Context Manager API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/accesscontextmanager/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/aiplatform/apiv1": { + "distribution_name": "cloud.google.com/go/aiplatform/apiv1", + "description": "Vertex AI API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/aiplatform/apiv1beta1": { + "distribution_name": "cloud.google.com/go/aiplatform/apiv1beta1", + "description": "Vertex AI API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/analytics/admin/apiv1alpha": { + "distribution_name": "cloud.google.com/go/analytics/admin/apiv1alpha", + "description": "Google Analytics Admin API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/analytics/latest/admin/apiv1alpha", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/apigateway/apiv1": { + "distribution_name": "cloud.google.com/go/apigateway/apiv1", + "description": "API Gateway API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigateway/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/apigeeconnect/apiv1": { + "distribution_name": "cloud.google.com/go/apigeeconnect/apiv1", + "description": "Apigee Connect API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigeeconnect/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/apigeeregistry/apiv1": { + "distribution_name": "cloud.google.com/go/apigeeregistry/apiv1", + "description": "Apigee Registry API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigeeregistry/latest/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/apikeys/apiv2": { + "distribution_name": "cloud.google.com/go/apikeys/apiv2", + "description": "API Keys API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apikeys/latest/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/appengine/apiv1": { + "distribution_name": "cloud.google.com/go/appengine/apiv1", + "description": "App Engine Admin API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/appengine/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/area120/tables/apiv1alpha1": { + "distribution_name": "cloud.google.com/go/area120/tables/apiv1alpha1", + "description": "Area120 Tables API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/area120/latest/tables/apiv1alpha1", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/artifactregistry/apiv1": { + "distribution_name": "cloud.google.com/go/artifactregistry/apiv1", + "description": "Artifact Registry API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/artifactregistry/latest/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/artifactregistry/apiv1beta2": { + "distribution_name": "cloud.google.com/go/artifactregistry/apiv1beta2", + "description": "Artifact Registry API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/artifactregistry/latest/apiv1beta2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/asset/apiv1": { + "distribution_name": "cloud.google.com/go/asset/apiv1", + "description": "Cloud Asset API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/asset/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/asset/apiv1p2beta1": { + "distribution_name": "cloud.google.com/go/asset/apiv1p2beta1", + "description": "Cloud Asset API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/asset/latest/apiv1p2beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/asset/apiv1p5beta1": { + "distribution_name": "cloud.google.com/go/asset/apiv1p5beta1", + "description": "Cloud Asset API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/asset/latest/apiv1p5beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/assuredworkloads/apiv1": { + "distribution_name": "cloud.google.com/go/assuredworkloads/apiv1", + "description": "Assured Workloads API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/assuredworkloads/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/assuredworkloads/apiv1beta1": { + "distribution_name": "cloud.google.com/go/assuredworkloads/apiv1beta1", + "description": "Assured Workloads API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/assuredworkloads/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/automl/apiv1": { + "distribution_name": "cloud.google.com/go/automl/apiv1", + "description": "Cloud AutoML API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/automl/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/automl/apiv1beta1": { + "distribution_name": "cloud.google.com/go/automl/apiv1beta1", + "description": "Cloud AutoML API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/automl/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/baremetalsolution/apiv2": { + "distribution_name": "cloud.google.com/go/baremetalsolution/apiv2", + "description": "Bare Metal Solution API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/baremetalsolution/latest/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/batch/apiv1": { + "distribution_name": "cloud.google.com/go/batch/apiv1", + "description": "Batch API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/batch/latest/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/beyondcorp/appconnections/apiv1": { + "distribution_name": "cloud.google.com/go/beyondcorp/appconnections/apiv1", + "description": "BeyondCorp API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/appconnections/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/beyondcorp/appconnectors/apiv1": { + "distribution_name": "cloud.google.com/go/beyondcorp/appconnectors/apiv1", + "description": "BeyondCorp API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/appconnectors/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/beyondcorp/appgateways/apiv1": { + "distribution_name": "cloud.google.com/go/beyondcorp/appgateways/apiv1", + "description": "BeyondCorp API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/appgateways/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/beyondcorp/clientconnectorservices/apiv1": { + "distribution_name": "cloud.google.com/go/beyondcorp/clientconnectorservices/apiv1", + "description": "BeyondCorp API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/clientconnectorservices/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/beyondcorp/clientgateways/apiv1": { + "distribution_name": "cloud.google.com/go/beyondcorp/clientgateways/apiv1", + "description": "BeyondCorp API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/clientgateways/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery": { + "distribution_name": "cloud.google.com/go/bigquery", + "description": "BigQuery", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/bigquery/analyticshub/apiv1": { + "distribution_name": "cloud.google.com/go/bigquery/analyticshub/apiv1", + "description": "Analytics Hub API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/analyticshub/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/connection/apiv1": { + "distribution_name": "cloud.google.com/go/bigquery/connection/apiv1", + "description": "BigQuery Connection API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/connection/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/connection/apiv1beta1": { + "distribution_name": "cloud.google.com/go/bigquery/connection/apiv1beta1", + "description": "BigQuery Connection API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/connection/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/dataexchange/apiv1beta1": { + "distribution_name": "cloud.google.com/go/bigquery/dataexchange/apiv1beta1", + "description": "Analytics Hub API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/dataexchange/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/datapolicies/apiv1beta1": { + "distribution_name": "cloud.google.com/go/bigquery/datapolicies/apiv1beta1", + "description": "BigQuery Data Policy API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/datapolicies/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/datatransfer/apiv1": { + "distribution_name": "cloud.google.com/go/bigquery/datatransfer/apiv1", + "description": "BigQuery Data Transfer API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/datatransfer/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/migration/apiv2": { + "distribution_name": "cloud.google.com/go/bigquery/migration/apiv2", + "description": "BigQuery Migration API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/migration/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/migration/apiv2alpha": { + "distribution_name": "cloud.google.com/go/bigquery/migration/apiv2alpha", + "description": "BigQuery Migration API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/migration/apiv2alpha", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/reservation/apiv1": { + "distribution_name": "cloud.google.com/go/bigquery/reservation/apiv1", + "description": "BigQuery Reservation API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/reservation/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/reservation/apiv1beta1": { + "distribution_name": "cloud.google.com/go/bigquery/reservation/apiv1beta1", + "description": "BigQuery Reservation API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/reservation/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/storage/apiv1": { + "distribution_name": "cloud.google.com/go/bigquery/storage/apiv1", + "description": "BigQuery Storage API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/storage/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/storage/apiv1beta1": { + "distribution_name": "cloud.google.com/go/bigquery/storage/apiv1beta1", + "description": "BigQuery Storage API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/storage/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigquery/storage/apiv1beta2": { + "distribution_name": "cloud.google.com/go/bigquery/storage/apiv1beta2", + "description": "BigQuery Storage API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/storage/apiv1beta2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/bigtable": { + "distribution_name": "cloud.google.com/go/bigtable", + "description": "Cloud BigTable", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigtable/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/billing/apiv1": { + "distribution_name": "cloud.google.com/go/billing/apiv1", + "description": "Cloud Billing API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/billing/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/billing/budgets/apiv1": { + "distribution_name": "cloud.google.com/go/billing/budgets/apiv1", + "description": "Cloud Billing Budget API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/billing/latest/budgets/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/billing/budgets/apiv1beta1": { + "distribution_name": "cloud.google.com/go/billing/budgets/apiv1beta1", + "description": "Cloud Billing Budget API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/billing/latest/budgets/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/binaryauthorization/apiv1": { + "distribution_name": "cloud.google.com/go/binaryauthorization/apiv1", + "description": "Binary Authorization API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/binaryauthorization/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/binaryauthorization/apiv1beta1": { + "distribution_name": "cloud.google.com/go/binaryauthorization/apiv1beta1", + "description": "Binary Authorization API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/binaryauthorization/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/certificatemanager/apiv1": { + "distribution_name": "cloud.google.com/go/certificatemanager/apiv1", + "description": "Certificate Manager API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/certificatemanager/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/channel/apiv1": { + "distribution_name": "cloud.google.com/go/channel/apiv1", + "description": "Cloud Channel API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/channel/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/cloudbuild/apiv1/v2": { + "distribution_name": "cloud.google.com/go/cloudbuild/apiv1/v2", + "description": "Cloud Build API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudbuild/latest/apiv1/v2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/clouddms/apiv1": { + "distribution_name": "cloud.google.com/go/clouddms/apiv1", + "description": "Database Migration API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/clouddms/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/cloudtasks/apiv2": { + "distribution_name": "cloud.google.com/go/cloudtasks/apiv2", + "description": "Cloud Tasks API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudtasks/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/cloudtasks/apiv2beta2": { + "distribution_name": "cloud.google.com/go/cloudtasks/apiv2beta2", + "description": "Cloud Tasks API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudtasks/latest/apiv2beta2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/cloudtasks/apiv2beta3": { + "distribution_name": "cloud.google.com/go/cloudtasks/apiv2beta3", + "description": "Cloud Tasks API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudtasks/latest/apiv2beta3", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/compute/apiv1": { + "distribution_name": "cloud.google.com/go/compute/apiv1", + "description": "Google Compute Engine API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/compute/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/compute/metadata": { + "distribution_name": "cloud.google.com/go/compute/metadata", + "description": "Service Metadata API", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/compute/latest/metadata", + "release_level": "ga", + "library_type": "CORE" + }, + "cloud.google.com/go/contactcenterinsights/apiv1": { + "distribution_name": "cloud.google.com/go/contactcenterinsights/apiv1", + "description": "Contact Center AI Insights API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/contactcenterinsights/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/container/apiv1": { + "distribution_name": "cloud.google.com/go/container/apiv1", + "description": "Kubernetes Engine API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/container/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/containeranalysis/apiv1beta1": { + "distribution_name": "cloud.google.com/go/containeranalysis/apiv1beta1", + "description": "Container Analysis API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/containeranalysis/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datacatalog/apiv1": { + "distribution_name": "cloud.google.com/go/datacatalog/apiv1", + "description": "Google Cloud Data Catalog API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datacatalog/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datacatalog/apiv1beta1": { + "distribution_name": "cloud.google.com/go/datacatalog/apiv1beta1", + "description": "Google Cloud Data Catalog API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datacatalog/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dataflow/apiv1beta3": { + "distribution_name": "cloud.google.com/go/dataflow/apiv1beta3", + "description": "Dataflow API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataflow/latest/apiv1beta3", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dataform/apiv1alpha2": { + "distribution_name": "cloud.google.com/go/dataform/apiv1alpha2", + "description": "Dataform API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataform/latest/apiv1alpha2", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dataform/apiv1beta1": { + "distribution_name": "cloud.google.com/go/dataform/apiv1beta1", + "description": "Dataform API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataform/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datafusion/apiv1": { + "distribution_name": "cloud.google.com/go/datafusion/apiv1", + "description": "Cloud Data Fusion API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datafusion/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datalabeling/apiv1beta1": { + "distribution_name": "cloud.google.com/go/datalabeling/apiv1beta1", + "description": "Data Labeling API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datalabeling/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dataplex/apiv1": { + "distribution_name": "cloud.google.com/go/dataplex/apiv1", + "description": "Cloud Dataplex API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataplex/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dataproc/apiv1": { + "distribution_name": "cloud.google.com/go/dataproc/apiv1", + "description": "Cloud Dataproc API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataproc/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dataqna/apiv1alpha": { + "distribution_name": "cloud.google.com/go/dataqna/apiv1alpha", + "description": "Data QnA API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataqna/latest/apiv1alpha", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datastore": { + "distribution_name": "cloud.google.com/go/datastore", + "description": "Cloud Datastore", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastore/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/datastore/admin/apiv1": { + "distribution_name": "cloud.google.com/go/datastore/admin/apiv1", + "description": "Cloud Datastore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastore/latest/admin/apiv1", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datastream/apiv1": { + "distribution_name": "cloud.google.com/go/datastream/apiv1", + "description": "Datastream API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastream/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/datastream/apiv1alpha1": { + "distribution_name": "cloud.google.com/go/datastream/apiv1alpha1", + "description": "Datastream API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastream/latest/apiv1alpha1", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/debugger/apiv2": { + "distribution_name": "cloud.google.com/go/debugger/apiv2", + "description": "Stackdriver Debugger API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/debugger/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/deploy/apiv1": { + "distribution_name": "cloud.google.com/go/deploy/apiv1", + "description": "Google Cloud Deploy API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/deploy/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dialogflow/apiv2": { + "distribution_name": "cloud.google.com/go/dialogflow/apiv2", + "description": "Dialogflow API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dialogflow/apiv2beta1": { + "distribution_name": "cloud.google.com/go/dialogflow/apiv2beta1", + "description": "Dialogflow API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/apiv2beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dialogflow/cx/apiv3": { + "distribution_name": "cloud.google.com/go/dialogflow/cx/apiv3", + "description": "Dialogflow API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/cx/apiv3", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dialogflow/cx/apiv3beta1": { + "distribution_name": "cloud.google.com/go/dialogflow/cx/apiv3beta1", + "description": "Dialogflow API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/cx/apiv3beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/dlp/apiv2": { + "distribution_name": "cloud.google.com/go/dlp/apiv2", + "description": "Cloud Data Loss Prevention (DLP) API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dlp/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/documentai/apiv1": { + "distribution_name": "cloud.google.com/go/documentai/apiv1", + "description": "Cloud Document AI API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/documentai/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/documentai/apiv1beta3": { + "distribution_name": "cloud.google.com/go/documentai/apiv1beta3", + "description": "Cloud Document AI API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/documentai/latest/apiv1beta3", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/domains/apiv1beta1": { + "distribution_name": "cloud.google.com/go/domains/apiv1beta1", + "description": "Cloud Domains API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/domains/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/edgecontainer/apiv1": { + "distribution_name": "cloud.google.com/go/edgecontainer/apiv1", + "description": "Distributed Cloud Edge Container API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/edgecontainer/latest/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/errorreporting": { + "distribution_name": "cloud.google.com/go/errorreporting", + "description": "Cloud Error Reporting API", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/errorreporting/latest", + "release_level": "beta", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/errorreporting/apiv1beta1": { + "distribution_name": "cloud.google.com/go/errorreporting/apiv1beta1", + "description": "Error Reporting API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/errorreporting/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/essentialcontacts/apiv1": { + "distribution_name": "cloud.google.com/go/essentialcontacts/apiv1", + "description": "Essential Contacts API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/essentialcontacts/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/eventarc/apiv1": { + "distribution_name": "cloud.google.com/go/eventarc/apiv1", + "description": "Eventarc API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/eventarc/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/eventarc/publishing/apiv1": { + "distribution_name": "cloud.google.com/go/eventarc/publishing/apiv1", + "description": "Eventarc Publishing API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/eventarc/latest/publishing/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/filestore/apiv1": { + "distribution_name": "cloud.google.com/go/filestore/apiv1", + "description": "Cloud Filestore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/filestore/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/firestore": { + "distribution_name": "cloud.google.com/go/firestore", + "description": "Cloud Firestore API", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/firestore/apiv1": { + "distribution_name": "cloud.google.com/go/firestore/apiv1", + "description": "Cloud Firestore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/firestore/apiv1/admin": { + "distribution_name": "cloud.google.com/go/firestore/apiv1/admin", + "description": "Cloud Firestore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest/apiv1/admin", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/functions/apiv1": { + "distribution_name": "cloud.google.com/go/functions/apiv1", + "description": "Cloud Functions API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/functions/apiv2": { + "distribution_name": "cloud.google.com/go/functions/apiv2", + "description": "Cloud Functions API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/functions/apiv2beta": { + "distribution_name": "cloud.google.com/go/functions/apiv2beta", + "description": "Cloud Functions API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/apiv2beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/functions/metadata": { + "distribution_name": "cloud.google.com/go/functions/metadata", + "description": "Cloud Functions", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/metadata", + "release_level": "alpha", + "library_type": "CORE" + }, + "cloud.google.com/go/gaming/apiv1": { + "distribution_name": "cloud.google.com/go/gaming/apiv1", + "description": "Game Services API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gaming/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/gaming/apiv1beta": { + "distribution_name": "cloud.google.com/go/gaming/apiv1beta", + "description": "Game Services API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gaming/latest/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/gkebackup/apiv1": { + "distribution_name": "cloud.google.com/go/gkebackup/apiv1", + "description": "Backup for GKE API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkebackup/latest/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/gkeconnect/gateway/apiv1beta1": { + "distribution_name": "cloud.google.com/go/gkeconnect/gateway/apiv1beta1", + "description": "Connect Gateway API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkeconnect/latest/gateway/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/gkehub/apiv1beta1": { + "distribution_name": "cloud.google.com/go/gkehub/apiv1beta1", + "description": "GKE Hub API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkehub/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/gkemulticloud/apiv1": { + "distribution_name": "cloud.google.com/go/gkemulticloud/apiv1", + "description": "Anthos Multi-Cloud API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkemulticloud/latest/apiv1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/gsuiteaddons/apiv1": { + "distribution_name": "cloud.google.com/go/gsuiteaddons/apiv1", + "description": "Google Workspace Add-ons API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gsuiteaddons/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/iam": { + "distribution_name": "cloud.google.com/go/iam", + "description": "Cloud IAM", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iam/latest", + "release_level": "ga", + "library_type": "CORE" + }, + "cloud.google.com/go/iam/apiv2": { + "distribution_name": "cloud.google.com/go/iam/apiv2", + "description": "Identity and Access Management (IAM) API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iam/latest/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/iam/credentials/apiv1": { + "distribution_name": "cloud.google.com/go/iam/credentials/apiv1", + "description": "IAM Service Account Credentials API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iam/latest/credentials/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/iap/apiv1": { + "distribution_name": "cloud.google.com/go/iap/apiv1", + "description": "Cloud Identity-Aware Proxy API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iap/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/ids/apiv1": { + "distribution_name": "cloud.google.com/go/ids/apiv1", + "description": "Cloud IDS API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/ids/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/iot/apiv1": { + "distribution_name": "cloud.google.com/go/iot/apiv1", + "description": "Cloud IoT API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iot/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/kms/apiv1": { + "distribution_name": "cloud.google.com/go/kms/apiv1", + "description": "Cloud Key Management Service (KMS) API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/kms/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/language/apiv1": { + "distribution_name": "cloud.google.com/go/language/apiv1", + "description": "Cloud Natural Language API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/language/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/language/apiv1beta2": { + "distribution_name": "cloud.google.com/go/language/apiv1beta2", + "description": "Cloud Natural Language API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/language/latest/apiv1beta2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/lifesciences/apiv2beta": { + "distribution_name": "cloud.google.com/go/lifesciences/apiv2beta", + "description": "Cloud Life Sciences API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/lifesciences/latest/apiv2beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/logging": { + "distribution_name": "cloud.google.com/go/logging", + "description": "Cloud Logging API", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/logging/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/logging/apiv2": { + "distribution_name": "cloud.google.com/go/logging/apiv2", + "description": "Cloud Logging API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/logging/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/longrunning/autogen": { + "distribution_name": "cloud.google.com/go/longrunning/autogen", + "description": "Long Running Operations API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/longrunning/autogen", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/managedidentities/apiv1": { + "distribution_name": "cloud.google.com/go/managedidentities/apiv1", + "description": "Managed Service for Microsoft Active Directory API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/managedidentities/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/mediatranslation/apiv1beta1": { + "distribution_name": "cloud.google.com/go/mediatranslation/apiv1beta1", + "description": "Media Translation API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/mediatranslation/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/memcache/apiv1": { + "distribution_name": "cloud.google.com/go/memcache/apiv1", + "description": "Cloud Memorystore for Memcached API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/memcache/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/memcache/apiv1beta2": { + "distribution_name": "cloud.google.com/go/memcache/apiv1beta2", + "description": "Cloud Memorystore for Memcached API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/memcache/latest/apiv1beta2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/metastore/apiv1": { + "distribution_name": "cloud.google.com/go/metastore/apiv1", + "description": "Dataproc Metastore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/metastore/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/metastore/apiv1alpha": { + "distribution_name": "cloud.google.com/go/metastore/apiv1alpha", + "description": "Dataproc Metastore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/metastore/latest/apiv1alpha", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/metastore/apiv1beta": { + "distribution_name": "cloud.google.com/go/metastore/apiv1beta", + "description": "Dataproc Metastore API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/metastore/latest/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/monitoring/apiv3/v2": { + "distribution_name": "cloud.google.com/go/monitoring/apiv3/v2", + "description": "Cloud Monitoring API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/monitoring/latest/apiv3/v2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/monitoring/dashboard/apiv1": { + "distribution_name": "cloud.google.com/go/monitoring/dashboard/apiv1", + "description": "Cloud Monitoring API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/monitoring/latest/dashboard/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/monitoring/metricsscope/apiv1": { + "distribution_name": "cloud.google.com/go/monitoring/metricsscope/apiv1", + "description": "Cloud Monitoring API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/monitoring/latest/metricsscope/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/networkconnectivity/apiv1": { + "distribution_name": "cloud.google.com/go/networkconnectivity/apiv1", + "description": "Network Connectivity API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networkconnectivity/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/networkconnectivity/apiv1alpha1": { + "distribution_name": "cloud.google.com/go/networkconnectivity/apiv1alpha1", + "description": "Network Connectivity API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networkconnectivity/latest/apiv1alpha1", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/networkmanagement/apiv1": { + "distribution_name": "cloud.google.com/go/networkmanagement/apiv1", + "description": "Network Management API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networkmanagement/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/networksecurity/apiv1beta1": { + "distribution_name": "cloud.google.com/go/networksecurity/apiv1beta1", + "description": "Network Security API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networksecurity/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/notebooks/apiv1": { + "distribution_name": "cloud.google.com/go/notebooks/apiv1", + "description": "Notebooks API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/notebooks/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/notebooks/apiv1beta1": { + "distribution_name": "cloud.google.com/go/notebooks/apiv1beta1", + "description": "Notebooks API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/notebooks/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/optimization/apiv1": { + "distribution_name": "cloud.google.com/go/optimization/apiv1", + "description": "Cloud Optimization API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/optimization/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/orchestration/airflow/service/apiv1": { + "distribution_name": "cloud.google.com/go/orchestration/airflow/service/apiv1", + "description": "Cloud Composer API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/orchestration/latest/airflow/service/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/orgpolicy/apiv2": { + "distribution_name": "cloud.google.com/go/orgpolicy/apiv2", + "description": "Organization Policy API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/orgpolicy/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/osconfig/agentendpoint/apiv1": { + "distribution_name": "cloud.google.com/go/osconfig/agentendpoint/apiv1", + "description": "OS Config API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/agentendpoint/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/osconfig/agentendpoint/apiv1beta": { + "distribution_name": "cloud.google.com/go/osconfig/agentendpoint/apiv1beta", + "description": "OS Config API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/agentendpoint/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/osconfig/apiv1": { + "distribution_name": "cloud.google.com/go/osconfig/apiv1", + "description": "OS Config API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/osconfig/apiv1alpha": { + "distribution_name": "cloud.google.com/go/osconfig/apiv1alpha", + "description": "OS Config API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/apiv1alpha", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/osconfig/apiv1beta": { + "distribution_name": "cloud.google.com/go/osconfig/apiv1beta", + "description": "OS Config API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/oslogin/apiv1": { + "distribution_name": "cloud.google.com/go/oslogin/apiv1", + "description": "Cloud OS Login API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/oslogin/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/oslogin/apiv1beta": { + "distribution_name": "cloud.google.com/go/oslogin/apiv1beta", + "description": "Cloud OS Login API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/oslogin/latest/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/phishingprotection/apiv1beta1": { + "distribution_name": "cloud.google.com/go/phishingprotection/apiv1beta1", + "description": "Phishing Protection API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/phishingprotection/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/policytroubleshooter/apiv1": { + "distribution_name": "cloud.google.com/go/policytroubleshooter/apiv1", + "description": "Policy Troubleshooter API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/policytroubleshooter/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/privatecatalog/apiv1beta1": { + "distribution_name": "cloud.google.com/go/privatecatalog/apiv1beta1", + "description": "Cloud Private Catalog API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/privatecatalog/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/profiler": { + "distribution_name": "cloud.google.com/go/profiler", + "description": "Cloud Profiler", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/profiler/latest", + "release_level": "ga", + "library_type": "AGENT" + }, + "cloud.google.com/go/pubsub": { + "distribution_name": "cloud.google.com/go/pubsub", + "description": "Cloud PubSub", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsub/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/pubsub/apiv1": { + "distribution_name": "cloud.google.com/go/pubsub/apiv1", + "description": "Cloud Pub/Sub API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsub/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/pubsublite": { + "distribution_name": "cloud.google.com/go/pubsublite", + "description": "Cloud PubSub Lite", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsublite/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/pubsublite/apiv1": { + "distribution_name": "cloud.google.com/go/pubsublite/apiv1", + "description": "Pub/Sub Lite API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsublite/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/recaptchaenterprise/v2/apiv1": { + "distribution_name": "cloud.google.com/go/recaptchaenterprise/v2/apiv1", + "description": "reCAPTCHA Enterprise API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recaptchaenterprise/v2/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1": { + "distribution_name": "cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1", + "description": "reCAPTCHA Enterprise API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recaptchaenterprise/v2/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/recommendationengine/apiv1beta1": { + "distribution_name": "cloud.google.com/go/recommendationengine/apiv1beta1", + "description": "Recommendations AI", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recommendationengine/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/recommender/apiv1": { + "distribution_name": "cloud.google.com/go/recommender/apiv1", + "description": "Recommender API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recommender/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/recommender/apiv1beta1": { + "distribution_name": "cloud.google.com/go/recommender/apiv1beta1", + "description": "Recommender API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recommender/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/redis/apiv1": { + "distribution_name": "cloud.google.com/go/redis/apiv1", + "description": "Google Cloud Memorystore for Redis API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/redis/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/redis/apiv1beta1": { + "distribution_name": "cloud.google.com/go/redis/apiv1beta1", + "description": "Google Cloud Memorystore for Redis API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/redis/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/resourcemanager/apiv2": { + "distribution_name": "cloud.google.com/go/resourcemanager/apiv2", + "description": "Cloud Resource Manager API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/resourcemanager/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/resourcemanager/apiv3": { + "distribution_name": "cloud.google.com/go/resourcemanager/apiv3", + "description": "Cloud Resource Manager API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/resourcemanager/latest/apiv3", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/resourcesettings/apiv1": { + "distribution_name": "cloud.google.com/go/resourcesettings/apiv1", + "description": "Resource Settings API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/resourcesettings/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/retail/apiv2": { + "distribution_name": "cloud.google.com/go/retail/apiv2", + "description": "Retail API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/retail/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/retail/apiv2alpha": { + "distribution_name": "cloud.google.com/go/retail/apiv2alpha", + "description": "Retail API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/retail/latest/apiv2alpha", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/retail/apiv2beta": { + "distribution_name": "cloud.google.com/go/retail/apiv2beta", + "description": "Retail API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/retail/latest/apiv2beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/rpcreplay": { + "distribution_name": "cloud.google.com/go/rpcreplay", + "description": "RPC Replay", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/rpcreplay", + "release_level": "ga", + "library_type": "OTHER" + }, + "cloud.google.com/go/run/apiv2": { + "distribution_name": "cloud.google.com/go/run/apiv2", + "description": "Cloud Run Admin API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/run/latest/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/scheduler/apiv1": { + "distribution_name": "cloud.google.com/go/scheduler/apiv1", + "description": "Cloud Scheduler API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/scheduler/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/scheduler/apiv1beta1": { + "distribution_name": "cloud.google.com/go/scheduler/apiv1beta1", + "description": "Cloud Scheduler API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/scheduler/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/secretmanager/apiv1": { + "distribution_name": "cloud.google.com/go/secretmanager/apiv1", + "description": "Secret Manager API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/secretmanager/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/security/privateca/apiv1": { + "distribution_name": "cloud.google.com/go/security/privateca/apiv1", + "description": "Certificate Authority API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/security/latest/privateca/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/security/privateca/apiv1beta1": { + "distribution_name": "cloud.google.com/go/security/privateca/apiv1beta1", + "description": "Certificate Authority API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/security/latest/privateca/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/security/publicca/apiv1beta1": { + "distribution_name": "cloud.google.com/go/security/publicca/apiv1beta1", + "description": "Public Certificate Authority API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/security/latest/publicca/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/securitycenter/apiv1": { + "distribution_name": "cloud.google.com/go/securitycenter/apiv1", + "description": "Security Command Center API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/securitycenter/apiv1beta1": { + "distribution_name": "cloud.google.com/go/securitycenter/apiv1beta1", + "description": "Security Command Center API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/securitycenter/apiv1p1beta1": { + "distribution_name": "cloud.google.com/go/securitycenter/apiv1p1beta1", + "description": "Security Command Center API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv1p1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/securitycenter/settings/apiv1beta1": { + "distribution_name": "cloud.google.com/go/securitycenter/settings/apiv1beta1", + "description": "Cloud Security Command Center API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/settings/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/servicecontrol/apiv1": { + "distribution_name": "cloud.google.com/go/servicecontrol/apiv1", + "description": "Service Control API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicecontrol/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/servicedirectory/apiv1": { + "distribution_name": "cloud.google.com/go/servicedirectory/apiv1", + "description": "Service Directory API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicedirectory/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/servicedirectory/apiv1beta1": { + "distribution_name": "cloud.google.com/go/servicedirectory/apiv1beta1", + "description": "Service Directory API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicedirectory/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/servicemanagement/apiv1": { + "distribution_name": "cloud.google.com/go/servicemanagement/apiv1", + "description": "Service Management API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicemanagement/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/serviceusage/apiv1": { + "distribution_name": "cloud.google.com/go/serviceusage/apiv1", + "description": "Service Usage API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/serviceusage/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/shell/apiv1": { + "distribution_name": "cloud.google.com/go/shell/apiv1", + "description": "Cloud Shell API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/shell/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/spanner": { + "distribution_name": "cloud.google.com/go/spanner", + "description": "Cloud Spanner", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/spanner/admin/database/apiv1": { + "distribution_name": "cloud.google.com/go/spanner/admin/database/apiv1", + "description": "Cloud Spanner API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/admin/database/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/spanner/admin/instance/apiv1": { + "distribution_name": "cloud.google.com/go/spanner/admin/instance/apiv1", + "description": "Cloud Spanner Instance Admin API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/admin/instance/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/spanner/apiv1": { + "distribution_name": "cloud.google.com/go/spanner/apiv1", + "description": "Cloud Spanner API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/speech/apiv1": { + "distribution_name": "cloud.google.com/go/speech/apiv1", + "description": "Cloud Speech-to-Text API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/speech/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/speech/apiv1p1beta1": { + "distribution_name": "cloud.google.com/go/speech/apiv1p1beta1", + "description": "Cloud Speech-to-Text API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/speech/latest/apiv1p1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/speech/apiv2": { + "distribution_name": "cloud.google.com/go/speech/apiv2", + "description": "Cloud Speech-to-Text API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/speech/latest/apiv2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/storage": { + "distribution_name": "cloud.google.com/go/storage", + "description": "Cloud Storage (GCS)", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest", + "release_level": "ga", + "library_type": "GAPIC_MANUAL" + }, + "cloud.google.com/go/storage/internal/apiv2": { + "distribution_name": "cloud.google.com/go/storage/internal/apiv2", + "description": "Cloud Storage API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest/internal/apiv2", + "release_level": "alpha", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/storagetransfer/apiv1": { + "distribution_name": "cloud.google.com/go/storagetransfer/apiv1", + "description": "Storage Transfer API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storagetransfer/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/talent/apiv4": { + "distribution_name": "cloud.google.com/go/talent/apiv4", + "description": "Cloud Talent Solution API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/talent/latest/apiv4", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/talent/apiv4beta1": { + "distribution_name": "cloud.google.com/go/talent/apiv4beta1", + "description": "Cloud Talent Solution API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/talent/latest/apiv4beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/texttospeech/apiv1": { + "distribution_name": "cloud.google.com/go/texttospeech/apiv1", + "description": "Cloud Text-to-Speech API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/texttospeech/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/tpu/apiv1": { + "distribution_name": "cloud.google.com/go/tpu/apiv1", + "description": "Cloud TPU API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/tpu/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/trace/apiv1": { + "distribution_name": "cloud.google.com/go/trace/apiv1", + "description": "Stackdriver Trace API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/trace/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/trace/apiv2": { + "distribution_name": "cloud.google.com/go/trace/apiv2", + "description": "Stackdriver Trace API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/trace/latest/apiv2", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/translate/apiv3": { + "distribution_name": "cloud.google.com/go/translate/apiv3", + "description": "Cloud Translation API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/translate/latest/apiv3", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/video/livestream/apiv1": { + "distribution_name": "cloud.google.com/go/video/livestream/apiv1", + "description": "Live Stream API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/livestream/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/video/stitcher/apiv1": { + "distribution_name": "cloud.google.com/go/video/stitcher/apiv1", + "description": "Video Stitcher API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/stitcher/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/video/transcoder/apiv1": { + "distribution_name": "cloud.google.com/go/video/transcoder/apiv1", + "description": "Transcoder API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/transcoder/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/videointelligence/apiv1": { + "distribution_name": "cloud.google.com/go/videointelligence/apiv1", + "description": "Cloud Video Intelligence API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/videointelligence/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/videointelligence/apiv1beta2": { + "distribution_name": "cloud.google.com/go/videointelligence/apiv1beta2", + "description": "Google Cloud Video Intelligence API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/videointelligence/latest/apiv1beta2", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/videointelligence/apiv1p3beta1": { + "distribution_name": "cloud.google.com/go/videointelligence/apiv1p3beta1", + "description": "Cloud Video Intelligence API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/videointelligence/latest/apiv1p3beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/vision/v2/apiv1": { + "distribution_name": "cloud.google.com/go/vision/v2/apiv1", + "description": "Cloud Vision API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vision/v2/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/vision/v2/apiv1p1beta1": { + "distribution_name": "cloud.google.com/go/vision/v2/apiv1p1beta1", + "description": "Cloud Vision API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vision/v2/latest/apiv1p1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/vmmigration/apiv1": { + "distribution_name": "cloud.google.com/go/vmmigration/apiv1", + "description": "VM Migration API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vmmigration/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/vpcaccess/apiv1": { + "distribution_name": "cloud.google.com/go/vpcaccess/apiv1", + "description": "Serverless VPC Access API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vpcaccess/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/webrisk/apiv1": { + "distribution_name": "cloud.google.com/go/webrisk/apiv1", + "description": "Web Risk API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/webrisk/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/webrisk/apiv1beta1": { + "distribution_name": "cloud.google.com/go/webrisk/apiv1beta1", + "description": "Web Risk API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/webrisk/latest/apiv1beta1", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/websecurityscanner/apiv1": { + "distribution_name": "cloud.google.com/go/websecurityscanner/apiv1", + "description": "Web Security Scanner API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/websecurityscanner/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/workflows/apiv1": { + "distribution_name": "cloud.google.com/go/workflows/apiv1", + "description": "Workflows API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/workflows/apiv1beta": { + "distribution_name": "cloud.google.com/go/workflows/apiv1beta", + "description": "Workflows API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/workflows/executions/apiv1": { + "distribution_name": "cloud.google.com/go/workflows/executions/apiv1", + "description": "Workflow Executions API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/executions/apiv1", + "release_level": "ga", + "library_type": "GAPIC_AUTO" + }, + "cloud.google.com/go/workflows/executions/apiv1beta": { + "distribution_name": "cloud.google.com/go/workflows/executions/apiv1beta", + "description": "Workflow Executions API", + "language": "Go", + "client_library_type": "generated", + "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/executions/apiv1beta", + "release_level": "beta", + "library_type": "GAPIC_AUTO" + } +} diff --git a/vendor/cloud.google.com/go/internal/README.md b/vendor/cloud.google.com/go/internal/README.md new file mode 100644 index 000000000..8857c8f6f --- /dev/null +++ b/vendor/cloud.google.com/go/internal/README.md @@ -0,0 +1,18 @@ +# Internal + +This directory contains internal code for cloud.google.com/go packages. + +## .repo-metadata-full.json + +`.repo-metadata-full.json` contains metadata about the packages in this repo. It +is generated by `internal/gapicgen/generator`. It's processed by external tools +to build lists of all of the packages. + +Don't make breaking changes to the format without consulting with the external +tools. + +One day, we may want to create individual `.repo-metadata.json` files next to +each package, which is the pattern followed by some other languages. External +tools would then talk to pkg.go.dev or some other service to get the overall +list of packages and use the `.repo-metadata.json` files to get the additional +metadata required. For now, `.repo-metadata-full.json` includes everything. \ No newline at end of file diff --git a/vendor/cloud.google.com/go/internal/annotate.go b/vendor/cloud.google.com/go/internal/annotate.go new file mode 100644 index 000000000..30d7bcf77 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/annotate.go @@ -0,0 +1,55 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "fmt" + + "google.golang.org/api/googleapi" + "google.golang.org/grpc/status" +) + +// Annotate prepends msg to the error message in err, attempting +// to preserve other information in err, like an error code. +// +// Annotate panics if err is nil. +// +// Annotate knows about these error types: +// - "google.golang.org/grpc/status".Status +// - "google.golang.org/api/googleapi".Error +// If the error is not one of these types, Annotate behaves +// like +// +// fmt.Errorf("%s: %v", msg, err) +func Annotate(err error, msg string) error { + if err == nil { + panic("Annotate called with nil") + } + if s, ok := status.FromError(err); ok { + p := s.Proto() + p.Message = msg + ": " + p.Message + return status.ErrorProto(p) + } + if g, ok := err.(*googleapi.Error); ok { + g.Message = msg + ": " + g.Message + return g + } + return fmt.Errorf("%s: %v", msg, err) +} + +// Annotatef uses format and args to format a string, then calls Annotate. +func Annotatef(err error, format string, args ...interface{}) error { + return Annotate(err, fmt.Sprintf(format, args...)) +} diff --git a/vendor/cloud.google.com/go/internal/optional/optional.go b/vendor/cloud.google.com/go/internal/optional/optional.go new file mode 100644 index 000000000..72780f764 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/optional/optional.go @@ -0,0 +1,108 @@ +// Copyright 2016 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package optional provides versions of primitive types that can +// be nil. These are useful in methods that update some of an API object's +// fields. +package optional + +import ( + "fmt" + "strings" + "time" +) + +type ( + // Bool is either a bool or nil. + Bool interface{} + + // String is either a string or nil. + String interface{} + + // Int is either an int or nil. + Int interface{} + + // Uint is either a uint or nil. + Uint interface{} + + // Float64 is either a float64 or nil. + Float64 interface{} + + // Duration is either a time.Duration or nil. + Duration interface{} +) + +// ToBool returns its argument as a bool. +// It panics if its argument is nil or not a bool. +func ToBool(v Bool) bool { + x, ok := v.(bool) + if !ok { + doPanic("Bool", v) + } + return x +} + +// ToString returns its argument as a string. +// It panics if its argument is nil or not a string. +func ToString(v String) string { + x, ok := v.(string) + if !ok { + doPanic("String", v) + } + return x +} + +// ToInt returns its argument as an int. +// It panics if its argument is nil or not an int. +func ToInt(v Int) int { + x, ok := v.(int) + if !ok { + doPanic("Int", v) + } + return x +} + +// ToUint returns its argument as a uint. +// It panics if its argument is nil or not a uint. +func ToUint(v Uint) uint { + x, ok := v.(uint) + if !ok { + doPanic("Uint", v) + } + return x +} + +// ToFloat64 returns its argument as a float64. +// It panics if its argument is nil or not a float64. +func ToFloat64(v Float64) float64 { + x, ok := v.(float64) + if !ok { + doPanic("Float64", v) + } + return x +} + +// ToDuration returns its argument as a time.Duration. +// It panics if its argument is nil or not a time.Duration. +func ToDuration(v Duration) time.Duration { + x, ok := v.(time.Duration) + if !ok { + doPanic("Duration", v) + } + return x +} + +func doPanic(capType string, v interface{}) { + panic(fmt.Sprintf("optional.%s value should be %s, got %T", capType, strings.ToLower(capType), v)) +} diff --git a/vendor/cloud.google.com/go/internal/retry.go b/vendor/cloud.google.com/go/internal/retry.go new file mode 100644 index 000000000..2943a6d0b --- /dev/null +++ b/vendor/cloud.google.com/go/internal/retry.go @@ -0,0 +1,85 @@ +// Copyright 2016 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "fmt" + "time" + + gax "github.com/googleapis/gax-go/v2" + "google.golang.org/grpc/status" +) + +// Retry calls the supplied function f repeatedly according to the provided +// backoff parameters. It returns when one of the following occurs: +// When f's first return value is true, Retry immediately returns with f's second +// return value. +// When the provided context is done, Retry returns with an error that +// includes both ctx.Error() and the last error returned by f. +func Retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error)) error { + return retry(ctx, bo, f, gax.Sleep) +} + +func retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error), + sleep func(context.Context, time.Duration) error) error { + var lastErr error + for { + stop, err := f() + if stop { + return err + } + // Remember the last "real" error from f. + if err != nil && err != context.Canceled && err != context.DeadlineExceeded { + lastErr = err + } + p := bo.Pause() + if ctxErr := sleep(ctx, p); ctxErr != nil { + if lastErr != nil { + return wrappedCallErr{ctxErr: ctxErr, wrappedErr: lastErr} + } + return ctxErr + } + } +} + +// Use this error type to return an error which allows introspection of both +// the context error and the error from the service. +type wrappedCallErr struct { + ctxErr error + wrappedErr error +} + +func (e wrappedCallErr) Error() string { + return fmt.Sprintf("retry failed with %v; last error: %v", e.ctxErr, e.wrappedErr) +} + +func (e wrappedCallErr) Unwrap() error { + return e.wrappedErr +} + +// Is allows errors.Is to match the error from the call as well as context +// sentinel errors. +func (e wrappedCallErr) Is(err error) bool { + return e.ctxErr == err || e.wrappedErr == err +} + +// GRPCStatus allows the wrapped error to be used with status.FromError. +func (e wrappedCallErr) GRPCStatus() *status.Status { + if s, ok := status.FromError(e.wrappedErr); ok { + return s + } + return nil +} diff --git a/vendor/cloud.google.com/go/internal/trace/trace.go b/vendor/cloud.google.com/go/internal/trace/trace.go new file mode 100644 index 000000000..c201d343e --- /dev/null +++ b/vendor/cloud.google.com/go/internal/trace/trace.go @@ -0,0 +1,111 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trace + +import ( + "context" + "fmt" + + "go.opencensus.io/trace" + "golang.org/x/xerrors" + "google.golang.org/api/googleapi" + "google.golang.org/genproto/googleapis/rpc/code" + "google.golang.org/grpc/status" +) + +// StartSpan adds a span to the trace with the given name. +func StartSpan(ctx context.Context, name string) context.Context { + ctx, _ = trace.StartSpan(ctx, name) + return ctx +} + +// EndSpan ends a span with the given error. +func EndSpan(ctx context.Context, err error) { + span := trace.FromContext(ctx) + if err != nil { + span.SetStatus(toStatus(err)) + } + span.End() +} + +// toStatus interrogates an error and converts it to an appropriate +// OpenCensus status. +func toStatus(err error) trace.Status { + var err2 *googleapi.Error + if ok := xerrors.As(err, &err2); ok { + return trace.Status{Code: httpStatusCodeToOCCode(err2.Code), Message: err2.Message} + } else if s, ok := status.FromError(err); ok { + return trace.Status{Code: int32(s.Code()), Message: s.Message()} + } else { + return trace.Status{Code: int32(code.Code_UNKNOWN), Message: err.Error()} + } +} + +// TODO(deklerk): switch to using OpenCensus function when it becomes available. +// Reference: https://github.com/googleapis/googleapis/blob/26b634d2724ac5dd30ae0b0cbfb01f07f2e4050e/google/rpc/code.proto +func httpStatusCodeToOCCode(httpStatusCode int) int32 { + switch httpStatusCode { + case 200: + return int32(code.Code_OK) + case 499: + return int32(code.Code_CANCELLED) + case 500: + return int32(code.Code_UNKNOWN) // Could also be Code_INTERNAL, Code_DATA_LOSS + case 400: + return int32(code.Code_INVALID_ARGUMENT) // Could also be Code_OUT_OF_RANGE + case 504: + return int32(code.Code_DEADLINE_EXCEEDED) + case 404: + return int32(code.Code_NOT_FOUND) + case 409: + return int32(code.Code_ALREADY_EXISTS) // Could also be Code_ABORTED + case 403: + return int32(code.Code_PERMISSION_DENIED) + case 401: + return int32(code.Code_UNAUTHENTICATED) + case 429: + return int32(code.Code_RESOURCE_EXHAUSTED) + case 501: + return int32(code.Code_UNIMPLEMENTED) + case 503: + return int32(code.Code_UNAVAILABLE) + default: + return int32(code.Code_UNKNOWN) + } +} + +// TODO: (odeke-em): perhaps just pass around spans due to the cost +// incurred from using trace.FromContext(ctx) yet we could avoid +// throwing away the work done by ctx, span := trace.StartSpan. +func TracePrintf(ctx context.Context, attrMap map[string]interface{}, format string, args ...interface{}) { + var attrs []trace.Attribute + for k, v := range attrMap { + var a trace.Attribute + switch v := v.(type) { + case string: + a = trace.StringAttribute(k, v) + case bool: + a = trace.BoolAttribute(k, v) + case int: + a = trace.Int64Attribute(k, int64(v)) + case int64: + a = trace.Int64Attribute(k, v) + default: + a = trace.StringAttribute(k, fmt.Sprintf("%#v", v)) + } + attrs = append(attrs, a) + } + trace.FromContext(ctx).Annotatef(attrs, format, args...) +} diff --git a/vendor/cloud.google.com/go/internal/version/update_version.sh b/vendor/cloud.google.com/go/internal/version/update_version.sh new file mode 100644 index 000000000..d7c5a3e21 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/version/update_version.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +today=$(date +%Y%m%d) + +sed -i -r -e 's/const Repo = "([0-9]{8})"/const Repo = "'$today'"/' $GOFILE + diff --git a/vendor/cloud.google.com/go/internal/version/version.go b/vendor/cloud.google.com/go/internal/version/version.go new file mode 100644 index 000000000..fd9dd91e9 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/version/version.go @@ -0,0 +1,71 @@ +// Copyright 2016 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate ./update_version.sh + +// Package version contains version information for Google Cloud Client +// Libraries for Go, as reported in request headers. +package version + +import ( + "runtime" + "strings" + "unicode" +) + +// Repo is the current version of the client libraries in this +// repo. It should be a date in YYYYMMDD format. +const Repo = "20201104" + +// Go returns the Go runtime version. The returned string +// has no whitespace. +func Go() string { + return goVersion +} + +var goVersion = goVer(runtime.Version()) + +const develPrefix = "devel +" + +func goVer(s string) string { + if strings.HasPrefix(s, develPrefix) { + s = s[len(develPrefix):] + if p := strings.IndexFunc(s, unicode.IsSpace); p >= 0 { + s = s[:p] + } + return s + } + + if strings.HasPrefix(s, "go1") { + s = s[2:] + var prerelease string + if p := strings.IndexFunc(s, notSemverRune); p >= 0 { + s, prerelease = s[:p], s[p:] + } + if strings.HasSuffix(s, ".") { + s += "0" + } else if strings.Count(s, ".") < 2 { + s += ".0" + } + if prerelease != "" { + s += "-" + prerelease + } + return s + } + return "" +} + +func notSemverRune(r rune) bool { + return !strings.ContainsRune("0123456789.", r) +} diff --git a/vendor/cloud.google.com/go/migration.md b/vendor/cloud.google.com/go/migration.md new file mode 100644 index 000000000..224dcfa13 --- /dev/null +++ b/vendor/cloud.google.com/go/migration.md @@ -0,0 +1,50 @@ +# go-genproto to google-cloud-go message type migration + +The message types for all of our client libraries are being migrated from the +`google.golang.org/genproto` [module](https://pkg.go.dev/google.golang.org/genproto) +to their respective product specific module in this repository. For example +this asset request type that was once found in [genproto](https://pkg.go.dev/google.golang.org/genproto@v0.0.0-20220908141613-51c1cc9bc6d0/googleapis/cloud/asset/v1p5beta1#ListAssetsRequest) +can now be found in directly in the [asset module](https://pkg.go.dev/cloud.google.com/go/asset/apiv1p5beta1/assetpb#ListAssetsRequest). + +Although the type definitions have moved, aliases have been left in the old +genproto packages to ensure a smooth non-breaking transition. + +## How do I migrate to the new packages? + +The easiest option is to run a migration tool at the root of our project. It is +like `go fix`, but specifically for this migration. Before running the tool it +is best to make sure any modules that have the prefix of `cloud.google.com/go` +are up to date. To run the tool, do the following: + +```bash +go run cloud.google.com/go/internal/aliasfix/cmd/aliasfix@latest . +go mod tidy +``` + +The tool should only change up to one line in the import statement per file. +This can also be done by hand if you prefer. + +## Do I have to migrate? + +Yes if you wish to keep using the newest versions of our client libraries with +the newest features -- You should migrate by the start of 2023. Until then we +will keep updating the aliases in go-genproto weekly. If you have an existing +workload that uses these client libraries and does not need to update its +dependencies there is no action to take. All existing written code will continue +to work. + +## Why are these types being moved + +1. This change will help simplify dependency trees over time. +2. The types will now be in product specific modules that are versioned + independently with semver. This is especially a benefit for users that rely + on multiple clients in a single application. Because message types are no + longer mono-packaged users are less likely to run into intermediate + dependency conflicts when updating dependencies. +3. Having all these types in one repository will help us ensure that unintended + changes are caught before they would be released. + +## Have questions? + +Please reach out to us on our [issue tracker](https://github.com/googleapis/google-cloud-go/issues/new?assignees=&labels=genproto-migration&template=migration-issue.md&title=package%3A+migration+help) +if you have any questions or concerns. diff --git a/vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json b/vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json new file mode 100644 index 000000000..172016990 --- /dev/null +++ b/vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json @@ -0,0 +1,341 @@ +{ + "release-type": "go-yoshi", + "include-component-in-tag": true, + "tag-separator": "/", + "packages": { + "accessapproval": { + "component": "accessapproval" + }, + "accesscontextmanager": { + "component": "accesscontextmanager" + }, + "aiplatform": { + "component": "aiplatform" + }, + "analytics": { + "component": "analytics" + }, + "apigateway": { + "component": "apigateway" + }, + "apigeeconnect": { + "component": "apigeeconnect" + }, + "apigeeregistry": { + "component": "apigeeregistry" + }, + "apikeys": { + "component": "apikeys" + }, + "appengine": { + "component": "appengine" + }, + "area120": { + "component": "area120" + }, + "artifactregistry": { + "component": "artifactregistry" + }, + "asset": { + "component": "asset" + }, + "assuredworkloads": { + "component": "assuredworkloads" + }, + "automl": { + "component": "automl" + }, + "baremetalsolution": { + "component": "baremetalsolution" + }, + "batch": { + "component": "batch" + }, + "beyondcorp": { + "component": "beyondcorp" + }, + "billing": { + "component": "billing" + }, + "binaryauthorization": { + "component": "binaryauthorization" + }, + "certificatemanager": { + "component": "certificatemanager" + }, + "channel": { + "component": "channel" + }, + "cloudbuild": { + "component": "cloudbuild" + }, + "clouddms": { + "component": "clouddms" + }, + "cloudtasks": { + "component": "cloudtasks" + }, + "compute": { + "component": "compute" + }, + "compute/metadata": { + "component": "compute/metadata" + }, + "contactcenterinsights": { + "component": "contactcenterinsights" + }, + "container": { + "component": "container" + }, + "containeranalysis": { + "component": "containeranalysis" + }, + "datacatalog": { + "component": "datacatalog" + }, + "dataflow": { + "component": "dataflow" + }, + "dataform": { + "component": "dataform" + }, + "datafusion": { + "component": "datafusion" + }, + "datalabeling": { + "component": "datalabeling" + }, + "dataplex": { + "component": "dataplex" + }, + "dataproc": { + "component": "dataproc" + }, + "dataqna": { + "component": "dataqna" + }, + "datastream": { + "component": "datastream" + }, + "deploy": { + "component": "deploy" + }, + "dialogflow": { + "component": "dialogflow" + }, + "dlp": { + "component": "dlp" + }, + "documentai": { + "component": "documentai" + }, + "domains": { + "component": "domains" + }, + "edgecontainer": { + "component": "edgecontainer" + }, + "essentialcontacts": { + "component": "essentialcontacts" + }, + "eventarc": { + "component": "eventarc" + }, + "filestore": { + "component": "filestore" + }, + "functions": { + "component": "functions" + }, + "gaming": { + "component": "gaming" + }, + "gkebackup": { + "component": "gkebackup" + }, + "gkeconnect": { + "component": "gkeconnect" + }, + "gkehub": { + "component": "gkehub" + }, + "gkemulticloud": { + "component": "gkemulticloud" + }, + "grafeas": { + "component": "grafeas" + }, + "gsuiteaddons": { + "component": "gsuiteaddons" + }, + "iam": { + "component": "iam" + }, + "iap": { + "component": "iap" + }, + "ids": { + "component": "ids" + }, + "iot": { + "component": "iot" + }, + "kms": { + "component": "kms" + }, + "language": { + "component": "language" + }, + "lifesciences": { + "component": "lifesciences" + }, + "managedidentities": { + "component": "managedidentities" + }, + "mediatranslation": { + "component": "mediatranslation" + }, + "memcache": { + "component": "memcache" + }, + "metastore": { + "component": "metastore" + }, + "monitoring": { + "component": "monitoring" + }, + "networkconnectivity": { + "component": "networkconnectivity" + }, + "networkmanagement": { + "component": "networkmanagement" + }, + "networksecurity": { + "component": "networksecurity" + }, + "notebooks": { + "component": "notebooks" + }, + "optimization": { + "component": "optimization" + }, + "orchestration": { + "component": "orchestration" + }, + "orgpolicy": { + "component": "orgpolicy" + }, + "osconfig": { + "component": "osconfig" + }, + "oslogin": { + "component": "oslogin" + }, + "phishingprotection": { + "component": "phishingprotection" + }, + "policytroubleshooter": { + "component": "policytroubleshooter" + }, + "privatecatalog": { + "component": "privatecatalog" + }, + "recaptchaenterprise/v2": { + "component": "recaptchaenterprise" + }, + "recommendationengine": { + "component": "recommendationengine" + }, + "recommender": { + "component": "recommender" + }, + "redis": { + "component": "redis" + }, + "resourcemanager": { + "component": "resourcemanager" + }, + "resourcesettings": { + "component": "resourcesettings" + }, + "retail": { + "component": "retail" + }, + "run": { + "component": "run" + }, + "scheduler": { + "component": "scheduler" + }, + "secretmanager": { + "component": "secretmanager" + }, + "security": { + "component": "security" + }, + "securitycenter": { + "component": "securitycenter" + }, + "servicecontrol": { + "component": "servicecontrol" + }, + "servicedirectory": { + "component": "servicedirectory" + }, + "servicemanagement": { + "component": "servicemanagement" + }, + "serviceusage": { + "component": "serviceusage" + }, + "shell": { + "component": "shell" + }, + "speech": { + "component": "speech" + }, + "storagetransfer": { + "component": "storagetransfer" + }, + "talent": { + "component": "talent" + }, + "texttospeech": { + "component": "texttospeech" + }, + "tpu": { + "component": "tpu" + }, + "trace": { + "component": "trace" + }, + "translate": { + "component": "translate" + }, + "video": { + "component": "video" + }, + "videointelligence": { + "component": "videointelligence" + }, + "vision/v2": { + "component": "vision" + }, + "vmmigration": { + "component": "vmmigration" + }, + "vpcaccess": { + "component": "vpcaccess" + }, + "webrisk": { + "component": "webrisk" + }, + "websecurityscanner": { + "component": "websecurityscanner" + }, + "workflows": { + "component": "workflows" + } + }, + "plugins": ["sentence-case"] +} diff --git a/vendor/cloud.google.com/go/release-please-config.json b/vendor/cloud.google.com/go/release-please-config.json new file mode 100644 index 000000000..1400245b8 --- /dev/null +++ b/vendor/cloud.google.com/go/release-please-config.json @@ -0,0 +1,11 @@ +{ + "release-type": "go-yoshi", + "separate-pull-requests": true, + "include-component-in-tag": false, + "packages": { + ".": { + "component": "main" + } + }, + "plugins": ["sentence-case"] +} diff --git a/vendor/cloud.google.com/go/testing.md b/vendor/cloud.google.com/go/testing.md new file mode 100644 index 000000000..bcca0604d --- /dev/null +++ b/vendor/cloud.google.com/go/testing.md @@ -0,0 +1,236 @@ +# Testing Code that depends on Go Client Libraries + +The Go client libraries generated as a part of `cloud.google.com/go` all take +the approach of returning concrete types instead of interfaces. That way, new +fields and methods can be added to the libraries without breaking users. This +document will go over some patterns that can be used to test code that depends +on the Go client libraries. + +## Testing gRPC services using fakes + +*Note*: You can see the full +[example code using a fake here](https://github.com/googleapis/google-cloud-go/tree/main/internal/examples/fake). + +The clients found in `cloud.google.com/go` are gRPC based, with a couple of +notable exceptions being the [`storage`](https://pkg.go.dev/cloud.google.com/go/storage) +and [`bigquery`](https://pkg.go.dev/cloud.google.com/go/bigquery) clients. +Interactions with gRPC services can be faked by serving up your own in-memory +server within your test. One benefit of using this approach is that you don’t +need to define an interface in your runtime code; you can keep using +concrete struct types. You instead define a fake server in your test code. For +example, take a look at the following function: + +```go +import ( + "context" + "fmt" + "log" + "os" + + translate "cloud.google.com/go/translate/apiv3" + "github.com/googleapis/gax-go/v2" + translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" +) + +func TranslateTextWithConcreteClient(client *translate.TranslationClient, text string, targetLang string) (string, error) { + ctx := context.Background() + log.Printf("Translating %q to %q", text, targetLang) + req := &translatepb.TranslateTextRequest{ + Parent: fmt.Sprintf("projects/%s/locations/global", os.Getenv("GOOGLE_CLOUD_PROJECT")), + TargetLanguageCode: "en-US", + Contents: []string{text}, + } + resp, err := client.TranslateText(ctx, req) + if err != nil { + return "", fmt.Errorf("unable to translate text: %v", err) + } + translations := resp.GetTranslations() + if len(translations) != 1 { + return "", fmt.Errorf("expected only one result, got %d", len(translations)) + } + return translations[0].TranslatedText, nil +} +``` + +Here is an example of what a fake server implementation would look like for +faking the interactions above: + +```go +import ( + "context" + + translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" +) + +type fakeTranslationServer struct { + translatepb.UnimplementedTranslationServiceServer +} + +func (f *fakeTranslationServer) TranslateText(ctx context.Context, req *translatepb.TranslateTextRequest) (*translatepb.TranslateTextResponse, error) { + resp := &translatepb.TranslateTextResponse{ + Translations: []*translatepb.Translation{ + &translatepb.Translation{ + TranslatedText: "Hello World", + }, + }, + } + return resp, nil +} +``` + +All of the generated protobuf code found in [google.golang.org/genproto](https://pkg.go.dev/google.golang.org/genproto) +contains a similar `package.UnimplmentedFooServer` type that is useful for +creating fakes. By embedding the unimplemented server in the +`fakeTranslationServer`, the fake will “inherit” all of the RPCs the server +exposes. Then, by providing our own `fakeTranslationServer.TranslateText` +method you can “override” the default unimplemented behavior of the one RPC that +you would like to be faked. + +The test itself does require a little bit of setup: start up a `net.Listener`, +register the server, and tell the client library to call the server: + +```go +import ( + "context" + "net" + "testing" + + translate "cloud.google.com/go/translate/apiv3" + "google.golang.org/api/option" + translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" + "google.golang.org/grpc" +) + +func TestTranslateTextWithConcreteClient(t *testing.T) { + ctx := context.Background() + + // Setup the fake server. + fakeTranslationServer := &fakeTranslationServer{} + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + gsrv := grpc.NewServer() + translatepb.RegisterTranslationServiceServer(gsrv, fakeTranslationServer) + fakeServerAddr := l.Addr().String() + go func() { + if err := gsrv.Serve(l); err != nil { + panic(err) + } + }() + + // Create a client. + client, err := translate.NewTranslationClient(ctx, + option.WithEndpoint(fakeServerAddr), + option.WithoutAuthentication(), + option.WithGRPCDialOption(grpc.WithInsecure()), + ) + if err != nil { + t.Fatal(err) + } + + // Run the test. + text, err := TranslateTextWithConcreteClient(client, "Hola Mundo", "en-US") + if err != nil { + t.Fatal(err) + } + if text != "Hello World" { + t.Fatalf("got %q, want Hello World", text) + } +} +``` + +## Testing using mocks + +*Note*: You can see the full +[example code using a mock here](https://github.com/googleapis/google-cloud-go/tree/main/internal/examples/mock). + +When mocking code you need to work with interfaces. Let’s create an interface +for the `cloud.google.com/go/translate/apiv3` client used in the +`TranslateTextWithConcreteClient` function mentioned in the previous section. +The `translate.Client` has over a dozen methods but this code only uses one of +them. Here is an interface that satisfies the interactions of the +`translate.Client` in this function. + +```go +type TranslationClient interface { + TranslateText(ctx context.Context, req *translatepb.TranslateTextRequest, opts ...gax.CallOption) (*translatepb.TranslateTextResponse, error) +} +``` + +Now that we have an interface that satisfies the method being used we can +rewrite the function signature to take the interface instead of the concrete +type. + +```go +func TranslateTextWithInterfaceClient(client TranslationClient, text string, targetLang string) (string, error) { +// ... +} +``` + +This allows a real `translate.Client` to be passed to the method in production +and for a mock implementation to be passed in during testing. This pattern can +be applied to any Go code, not just `cloud.google.com/go`. This is because +interfaces in Go are implicitly satisfied. Structs in the client libraries can +implicitly implement interfaces defined in your codebase. Let’s take a look at +what it might look like to define a lightweight mock for the `TranslationClient` +interface. + +```go +import ( + "context" + "testing" + + "github.com/googleapis/gax-go/v2" + translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" +) + +type mockClient struct{} + +func (*mockClient) TranslateText(_ context.Context, req *translatepb.TranslateTextRequest, opts ...gax.CallOption) (*translatepb.TranslateTextResponse, error) { + resp := &translatepb.TranslateTextResponse{ + Translations: []*translatepb.Translation{ + &translatepb.Translation{ + TranslatedText: "Hello World", + }, + }, + } + return resp, nil +} + +func TestTranslateTextWithAbstractClient(t *testing.T) { + client := &mockClient{} + text, err := TranslateTextWithInterfaceClient(client, "Hola Mundo", "en-US") + if err != nil { + t.Fatal(err) + } + if text != "Hello World" { + t.Fatalf("got %q, want Hello World", text) + } +} +``` + +If you prefer to not write your own mocks there are mocking frameworks such as +[golang/mock](https://github.com/golang/mock) which can generate mocks for you +from an interface. As a word of caution though, try to not +[overuse mocks](https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html). + +## Testing using emulators + +Some of the client libraries provided in `cloud.google.com/go` support running +against a service emulator. The concept is similar to that of using fakes, +mentioned above, but the server is managed for you. You just need to start it up +and instruct the client library to talk to the emulator by setting a service +specific emulator environment variable. Current services/environment-variables +are: + +- bigtable: `BIGTABLE_EMULATOR_HOST` +- datastore: `DATASTORE_EMULATOR_HOST` +- firestore: `FIRESTORE_EMULATOR_HOST` +- pubsub: `PUBSUB_EMULATOR_HOST` +- spanner: `SPANNER_EMULATOR_HOST` +- storage: `STORAGE_EMULATOR_HOST` + - Although the storage client supports an emulator environment variable there is no official emulator provided by gcloud. + +For more information on emulators please refer to the +[gcloud documentation](https://cloud.google.com/sdk/gcloud/reference/beta/emulators). From 1930260ed0dc63cb29f44c63a2753698b76c9e93 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 2 Apr 2024 21:28:28 +0530 Subject: [PATCH 127/235] Sentinel webhook uses mounted serviceaccount token for `ReconcilerServiceAccount` by default --- go.mod | 1 + go.sum | 2 + internal/webhook/sentinel/config.go | 48 ++- .../github.com/golang-jwt/jwt/v5/.gitignore | 4 + vendor/github.com/golang-jwt/jwt/v5/LICENSE | 9 + .../golang-jwt/jwt/v5/MIGRATION_GUIDE.md | 195 +++++++++++ vendor/github.com/golang-jwt/jwt/v5/README.md | 167 +++++++++ .../github.com/golang-jwt/jwt/v5/SECURITY.md | 19 ++ .../golang-jwt/jwt/v5/VERSION_HISTORY.md | 137 ++++++++ vendor/github.com/golang-jwt/jwt/v5/claims.go | 16 + vendor/github.com/golang-jwt/jwt/v5/doc.go | 4 + vendor/github.com/golang-jwt/jwt/v5/ecdsa.go | 134 ++++++++ .../golang-jwt/jwt/v5/ecdsa_utils.go | 69 ++++ .../github.com/golang-jwt/jwt/v5/ed25519.go | 79 +++++ .../golang-jwt/jwt/v5/ed25519_utils.go | 64 ++++ vendor/github.com/golang-jwt/jwt/v5/errors.go | 49 +++ .../golang-jwt/jwt/v5/errors_go1_20.go | 47 +++ .../golang-jwt/jwt/v5/errors_go_other.go | 78 +++++ vendor/github.com/golang-jwt/jwt/v5/hmac.go | 104 ++++++ .../golang-jwt/jwt/v5/map_claims.go | 109 ++++++ vendor/github.com/golang-jwt/jwt/v5/none.go | 50 +++ vendor/github.com/golang-jwt/jwt/v5/parser.go | 238 +++++++++++++ .../golang-jwt/jwt/v5/parser_option.go | 128 +++++++ .../golang-jwt/jwt/v5/registered_claims.go | 63 ++++ vendor/github.com/golang-jwt/jwt/v5/rsa.go | 93 ++++++ .../github.com/golang-jwt/jwt/v5/rsa_pss.go | 135 ++++++++ .../github.com/golang-jwt/jwt/v5/rsa_utils.go | 107 ++++++ .../golang-jwt/jwt/v5/signing_method.go | 49 +++ .../golang-jwt/jwt/v5/staticcheck.conf | 1 + vendor/github.com/golang-jwt/jwt/v5/token.go | 100 ++++++ .../golang-jwt/jwt/v5/token_option.go | 5 + vendor/github.com/golang-jwt/jwt/v5/types.go | 149 +++++++++ .../github.com/golang-jwt/jwt/v5/validator.go | 316 ++++++++++++++++++ 33 files changed, 2768 insertions(+), 1 deletion(-) create mode 100644 vendor/github.com/golang-jwt/jwt/v5/.gitignore create mode 100644 vendor/github.com/golang-jwt/jwt/v5/LICENSE create mode 100644 vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md create mode 100644 vendor/github.com/golang-jwt/jwt/v5/README.md create mode 100644 vendor/github.com/golang-jwt/jwt/v5/SECURITY.md create mode 100644 vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md create mode 100644 vendor/github.com/golang-jwt/jwt/v5/claims.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/doc.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/ecdsa.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/ed25519.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/errors.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/hmac.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/map_claims.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/none.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/parser.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/parser_option.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/registered_claims.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/rsa.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/signing_method.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf create mode 100644 vendor/github.com/golang-jwt/jwt/v5/token.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/token_option.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/types.go create mode 100644 vendor/github.com/golang-jwt/jwt/v5/validator.go diff --git a/go.mod b/go.mod index 6790cba16..4e3377c6b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/gardener/etcd-backup-restore v0.26.0 github.com/gardener/gardener v1.86.0 github.com/go-logr/logr v1.2.4 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/go.sum b/go.sum index df3256986..f6501f1e9 100644 --- a/go.sum +++ b/go.sum @@ -195,6 +195,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/sentinel/config.go index 51c165fcf..8cdee0ba3 100644 --- a/internal/webhook/sentinel/config.go +++ b/internal/webhook/sentinel/config.go @@ -5,8 +5,13 @@ package sentinel import ( + "encoding/base64" + "encoding/json" "fmt" + "os" + "strings" + "github.com/golang-jwt/jwt/v5" flag "github.com/spf13/pflag" ) @@ -17,6 +22,8 @@ const ( defaultEnableSentinelWebhook = false defaultReconcilerServiceAccount = "system:serviceaccount:default:etcd-druid" + + reconcilerServiceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) var ( @@ -37,8 +44,47 @@ type Config struct { func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.BoolVar(&cfg.Enabled, enableSentinelWebhookFlagName, defaultEnableSentinelWebhook, "Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid.") - fs.StringVar(&cfg.ReconcilerServiceAccount, reconcilerServiceAccountFlagName, defaultReconcilerServiceAccount, + + reconcilerServiceAccount, err := getReconcilerServiceAccountName() + if err != nil { + reconcilerServiceAccount = defaultReconcilerServiceAccount + } + fs.StringVar(&cfg.ReconcilerServiceAccount, reconcilerServiceAccountFlagName, reconcilerServiceAccount, fmt.Sprintf("The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. Default: %s", defaultReconcilerServiceAccount)) + fs.StringSliceVar(&cfg.ExemptServiceAccounts, exemptServiceAccountsFlagName, defaultExemptServiceAccounts, "The comma-separated list of fully qualified names of service accounts that are exempt from Sentinel Webhook checks.") } + +func getReconcilerServiceAccountName() (string, error) { + saToken, err := os.ReadFile(reconcilerServiceAccountTokenPath) + if err != nil { + return "", err + } + tokens := strings.Split(string(saToken), ".") + if len(tokens) != 3 { + return "", fmt.Errorf("invalid token format") + } + + decodedClaims, err := base64.StdEncoding.DecodeString(getPaddedBase64EncodedString(tokens[1])) + if err != nil { + return "", err + } + + claims := &jwt.RegisteredClaims{} + if err = json.Unmarshal(decodedClaims, claims); err != nil { + return "", err + } + + return claims.Subject, nil +} + +func getPaddedBase64EncodedString(encoded string) string { + padding := 4 - len(encoded)%4 + if padding != 4 { + for i := 0; i < padding; i++ { + encoded += "=" + } + } + return encoded +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/.gitignore b/vendor/github.com/golang-jwt/jwt/v5/.gitignore new file mode 100644 index 000000000..09573e016 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +bin +.idea/ + diff --git a/vendor/github.com/golang-jwt/jwt/v5/LICENSE b/vendor/github.com/golang-jwt/jwt/v5/LICENSE new file mode 100644 index 000000000..35dbc2520 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) 2012 Dave Grijalva +Copyright (c) 2021 golang-jwt maintainers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md b/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md new file mode 100644 index 000000000..ff9c57e1d --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md @@ -0,0 +1,195 @@ +# Migration Guide (v5.0.0) + +Version `v5` contains a major rework of core functionalities in the `jwt-go` +library. This includes support for several validation options as well as a +re-design of the `Claims` interface. Lastly, we reworked how errors work under +the hood, which should provide a better overall developer experience. + +Starting from [v5.0.0](https://github.com/golang-jwt/jwt/releases/tag/v5.0.0), +the import path will be: + + "github.com/golang-jwt/jwt/v5" + +For most users, changing the import path *should* suffice. However, since we +intentionally changed and cleaned some of the public API, existing programs +might need to be updated. The following sections describe significant changes +and corresponding updates for existing programs. + +## Parsing and Validation Options + +Under the hood, a new `Validator` struct takes care of validating the claims. A +long awaited feature has been the option to fine-tune the validation of tokens. +This is now possible with several `ParserOption` functions that can be appended +to most `Parse` functions, such as `ParseWithClaims`. The most important options +and changes are: + * Added `WithLeeway` to support specifying the leeway that is allowed when + validating time-based claims, such as `exp` or `nbf`. + * Changed default behavior to not check the `iat` claim. Usage of this claim + is OPTIONAL according to the JWT RFC. The claim itself is also purely + informational according to the RFC, so a strict validation failure is not + recommended. If you want to check for sensible values in these claims, + please use the `WithIssuedAt` parser option. + * Added `WithAudience`, `WithSubject` and `WithIssuer` to support checking for + expected `aud`, `sub` and `iss`. + * Added `WithStrictDecoding` and `WithPaddingAllowed` options to allow + previously global settings to enable base64 strict encoding and the parsing + of base64 strings with padding. The latter is strictly speaking against the + standard, but unfortunately some of the major identity providers issue some + of these incorrect tokens. Both options are disabled by default. + +## Changes to the `Claims` interface + +### Complete Restructuring + +Previously, the claims interface was satisfied with an implementation of a +`Valid() error` function. This had several issues: + * The different claim types (struct claims, map claims, etc.) then contained + similar (but not 100 % identical) code of how this validation was done. This + lead to a lot of (almost) duplicate code and was hard to maintain + * It was not really semantically close to what a "claim" (or a set of claims) + really is; which is a list of defined key/value pairs with a certain + semantic meaning. + +Since all the validation functionality is now extracted into the validator, all +`VerifyXXX` and `Valid` functions have been removed from the `Claims` interface. +Instead, the interface now represents a list of getters to retrieve values with +a specific meaning. This allows us to completely decouple the validation logic +with the underlying storage representation of the claim, which could be a +struct, a map or even something stored in a database. + +```go +type Claims interface { + GetExpirationTime() (*NumericDate, error) + GetIssuedAt() (*NumericDate, error) + GetNotBefore() (*NumericDate, error) + GetIssuer() (string, error) + GetSubject() (string, error) + GetAudience() (ClaimStrings, error) +} +``` + +Users that previously directly called the `Valid` function on their claims, +e.g., to perform validation independently of parsing/verifying a token, can now +use the `jwt.NewValidator` function to create a `Validator` independently of the +`Parser`. + +```go +var v = jwt.NewValidator(jwt.WithLeeway(5*time.Second)) +v.Validate(myClaims) +``` + +### Supported Claim Types and Removal of `StandardClaims` + +The two standard claim types supported by this library, `MapClaims` and +`RegisteredClaims` both implement the necessary functions of this interface. The +old `StandardClaims` struct, which has already been deprecated in `v4` is now +removed. + +Users using custom claims, in most cases, will not experience any changes in the +behavior as long as they embedded `RegisteredClaims`. If they created a new +claim type from scratch, they now need to implemented the proper getter +functions. + +### Migrating Application Specific Logic of the old `Valid` + +Previously, users could override the `Valid` method in a custom claim, for +example to extend the validation with application-specific claims. However, this +was always very dangerous, since once could easily disable the standard +validation and signature checking. + +In order to avoid that, while still supporting the use-case, a new +`ClaimsValidator` interface has been introduced. This interface consists of the +`Validate() error` function. If the validator sees, that a `Claims` struct +implements this interface, the errors returned to the `Validate` function will +be *appended* to the regular standard validation. It is not possible to disable +the standard validation anymore (even only by accident). + +Usage examples can be found in [example_test.go](./example_test.go), to build +claims structs like the following. + +```go +// MyCustomClaims includes all registered claims, plus Foo. +type MyCustomClaims struct { + Foo string `json:"foo"` + jwt.RegisteredClaims +} + +// Validate can be used to execute additional application-specific claims +// validation. +func (m MyCustomClaims) Validate() error { + if m.Foo != "bar" { + return errors.New("must be foobar") + } + + return nil +} +``` + +## Changes to the `Token` and `Parser` struct + +The previously global functions `DecodeSegment` and `EncodeSegment` were moved +to the `Parser` and `Token` struct respectively. This will allow us in the +future to configure the behavior of these two based on options supplied on the +parser or the token (creation). This also removes two previously global +variables and moves them to parser options `WithStrictDecoding` and +`WithPaddingAllowed`. + +In order to do that, we had to adjust the way signing methods work. Previously +they were given a base64 encoded signature in `Verify` and were expected to +return a base64 encoded version of the signature in `Sign`, both as a `string`. +However, this made it necessary to have `DecodeSegment` and `EncodeSegment` +global and was a less than perfect design because we were repeating +encoding/decoding steps for all signing methods. Now, `Sign` and `Verify` +operate on a decoded signature as a `[]byte`, which feels more natural for a +cryptographic operation anyway. Lastly, `Parse` and `SignedString` take care of +the final encoding/decoding part. + +In addition to that, we also changed the `Signature` field on `Token` from a +`string` to `[]byte` and this is also now populated with the decoded form. This +is also more consistent, because the other parts of the JWT, mainly `Header` and +`Claims` were already stored in decoded form in `Token`. Only the signature was +stored in base64 encoded form, which was redundant with the information in the +`Raw` field, which contains the complete token as base64. + +```go +type Token struct { + Raw string // Raw contains the raw token + Method SigningMethod // Method is the signing method used or to be used + Header map[string]interface{} // Header is the first segment of the token in decoded form + Claims Claims // Claims is the second segment of the token in decoded form + Signature []byte // Signature is the third segment of the token in decoded form + Valid bool // Valid specifies if the token is valid +} +``` + +Most (if not all) of these changes should not impact the normal usage of this +library. Only users directly accessing the `Signature` field as well as +developers of custom signing methods should be affected. + +# Migration Guide (v4.0.0) + +Starting from [v4.0.0](https://github.com/golang-jwt/jwt/releases/tag/v4.0.0), +the import path will be: + + "github.com/golang-jwt/jwt/v4" + +The `/v4` version will be backwards compatible with existing `v3.x.y` tags in +this repo, as well as `github.com/dgrijalva/jwt-go`. For most users this should +be a drop-in replacement, if you're having troubles migrating, please open an +issue. + +You can replace all occurrences of `github.com/dgrijalva/jwt-go` or +`github.com/golang-jwt/jwt` with `github.com/golang-jwt/jwt/v4`, either manually +or by using tools such as `sed` or `gofmt`. + +And then you'd typically run: + +``` +go get github.com/golang-jwt/jwt/v4 +go mod tidy +``` + +# Older releases (before v3.2.0) + +The original migration guide for older releases can be found at +https://github.com/dgrijalva/jwt-go/blob/master/MIGRATION_GUIDE.md. diff --git a/vendor/github.com/golang-jwt/jwt/v5/README.md b/vendor/github.com/golang-jwt/jwt/v5/README.md new file mode 100644 index 000000000..964598a31 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/README.md @@ -0,0 +1,167 @@ +# jwt-go + +[![build](https://github.com/golang-jwt/jwt/actions/workflows/build.yml/badge.svg)](https://github.com/golang-jwt/jwt/actions/workflows/build.yml) +[![Go +Reference](https://pkg.go.dev/badge/github.com/golang-jwt/jwt/v5.svg)](https://pkg.go.dev/github.com/golang-jwt/jwt/v5) +[![Coverage Status](https://coveralls.io/repos/github/golang-jwt/jwt/badge.svg?branch=main)](https://coveralls.io/github/golang-jwt/jwt?branch=main) + +A [go](http://www.golang.org) (or 'golang' for search engine friendliness) +implementation of [JSON Web +Tokens](https://datatracker.ietf.org/doc/html/rfc7519). + +Starting with [v4.0.0](https://github.com/golang-jwt/jwt/releases/tag/v4.0.0) +this project adds Go module support, but maintains backwards compatibility with +older `v3.x.y` tags and upstream `github.com/dgrijalva/jwt-go`. See the +[`MIGRATION_GUIDE.md`](./MIGRATION_GUIDE.md) for more information. Version +v5.0.0 introduces major improvements to the validation of tokens, but is not +entirely backwards compatible. + +> After the original author of the library suggested migrating the maintenance +> of `jwt-go`, a dedicated team of open source maintainers decided to clone the +> existing library into this repository. See +> [dgrijalva/jwt-go#462](https://github.com/dgrijalva/jwt-go/issues/462) for a +> detailed discussion on this topic. + + +**SECURITY NOTICE:** Some older versions of Go have a security issue in the +crypto/elliptic. Recommendation is to upgrade to at least 1.15 See issue +[dgrijalva/jwt-go#216](https://github.com/dgrijalva/jwt-go/issues/216) for more +detail. + +**SECURITY NOTICE:** It's important that you [validate the `alg` presented is +what you +expect](https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/). +This library attempts to make it easy to do the right thing by requiring key +types match the expected alg, but you should take the extra step to verify it in +your usage. See the examples provided. + +### Supported Go versions + +Our support of Go versions is aligned with Go's [version release +policy](https://golang.org/doc/devel/release#policy). So we will support a major +version of Go until there are two newer major releases. We no longer support +building jwt-go with unsupported Go versions, as these contain security +vulnerabilities which will not be fixed. + +## What the heck is a JWT? + +JWT.io has [a great introduction](https://jwt.io/introduction) to JSON Web +Tokens. + +In short, it's a signed JSON object that does something useful (for example, +authentication). It's commonly used for `Bearer` tokens in Oauth 2. A token is +made of three parts, separated by `.`'s. The first two parts are JSON objects, +that have been [base64url](https://datatracker.ietf.org/doc/html/rfc4648) +encoded. The last part is the signature, encoded the same way. + +The first part is called the header. It contains the necessary information for +verifying the last part, the signature. For example, which encryption method +was used for signing and what key was used. + +The part in the middle is the interesting bit. It's called the Claims and +contains the actual stuff you care about. Refer to [RFC +7519](https://datatracker.ietf.org/doc/html/rfc7519) for information about +reserved keys and the proper way to add your own. + +## What's in the box? + +This library supports the parsing and verification as well as the generation and +signing of JWTs. Current supported signing algorithms are HMAC SHA, RSA, +RSA-PSS, and ECDSA, though hooks are present for adding your own. + +## Installation Guidelines + +1. To install the jwt package, you first need to have + [Go](https://go.dev/doc/install) installed, then you can use the command + below to add `jwt-go` as a dependency in your Go program. + +```sh +go get -u github.com/golang-jwt/jwt/v5 +``` + +2. Import it in your code: + +```go +import "github.com/golang-jwt/jwt/v5" +``` + +## Usage + +A detailed usage guide, including how to sign and verify tokens can be found on +our [documentation website](https://golang-jwt.github.io/jwt/usage/create/). + +## Examples + +See [the project documentation](https://pkg.go.dev/github.com/golang-jwt/jwt/v5) +for examples of usage: + +* [Simple example of parsing and validating a + token](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#example-Parse-Hmac) +* [Simple example of building and signing a + token](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#example-New-Hmac) +* [Directory of + Examples](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#pkg-examples) + +## Compliance + +This library was last reviewed to comply with [RFC +7519](https://datatracker.ietf.org/doc/html/rfc7519) dated May 2015 with a few +notable differences: + +* In order to protect against accidental use of [Unsecured + JWTs](https://datatracker.ietf.org/doc/html/rfc7519#section-6), tokens using + `alg=none` will only be accepted if the constant + `jwt.UnsafeAllowNoneSignatureType` is provided as the key. + +## Project Status & Versioning + +This library is considered production ready. Feedback and feature requests are +appreciated. The API should be considered stable. There should be very few +backwards-incompatible changes outside of major version updates (and only with +good reason). + +This project uses [Semantic Versioning 2.0.0](http://semver.org). Accepted pull +requests will land on `main`. Periodically, versions will be tagged from +`main`. You can find all the releases on [the project releases +page](https://github.com/golang-jwt/jwt/releases). + +**BREAKING CHANGES:*** A full list of breaking changes is available in +`VERSION_HISTORY.md`. See `MIGRATION_GUIDE.md` for more information on updating +your code. + +## Extensions + +This library publishes all the necessary components for adding your own signing +methods or key functions. Simply implement the `SigningMethod` interface and +register a factory method using `RegisterSigningMethod` or provide a +`jwt.Keyfunc`. + +A common use case would be integrating with different 3rd party signature +providers, like key management services from various cloud providers or Hardware +Security Modules (HSMs) or to implement additional standards. + +| Extension | Purpose | Repo | +| --------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------ | +| GCP | Integrates with multiple Google Cloud Platform signing tools (AppEngine, IAM API, Cloud KMS) | https://github.com/someone1/gcp-jwt-go | +| AWS | Integrates with AWS Key Management Service, KMS | https://github.com/matelang/jwt-go-aws-kms | +| JWKS | Provides support for JWKS ([RFC 7517](https://datatracker.ietf.org/doc/html/rfc7517)) as a `jwt.Keyfunc` | https://github.com/MicahParks/keyfunc | + +*Disclaimer*: Unless otherwise specified, these integrations are maintained by +third parties and should not be considered as a primary offer by any of the +mentioned cloud providers + +## More + +Go package documentation can be found [on +pkg.go.dev](https://pkg.go.dev/github.com/golang-jwt/jwt/v5). Additional +documentation can be found on [our project +page](https://golang-jwt.github.io/jwt/). + +The command line utility included in this project (cmd/jwt) provides a +straightforward example of token creation and parsing as well as a useful tool +for debugging your own integration. You'll also find several implementation +examples in the documentation. + +[golang-jwt](https://github.com/orgs/golang-jwt) incorporates a modified version +of the JWT logo, which is distributed under the terms of the [MIT +License](https://github.com/jsonwebtoken/jsonwebtoken.github.io/blob/master/LICENSE.txt). diff --git a/vendor/github.com/golang-jwt/jwt/v5/SECURITY.md b/vendor/github.com/golang-jwt/jwt/v5/SECURITY.md new file mode 100644 index 000000000..b08402c34 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/SECURITY.md @@ -0,0 +1,19 @@ +# Security Policy + +## Supported Versions + +As of February 2022 (and until this document is updated), the latest version `v4` is supported. + +## Reporting a Vulnerability + +If you think you found a vulnerability, and even if you are not sure, please report it to jwt-go-security@googlegroups.com or one of the other [golang-jwt maintainers](https://github.com/orgs/golang-jwt/people). Please try be explicit, describe steps to reproduce the security issue with code example(s). + +You will receive a response within a timely manner. If the issue is confirmed, we will do our best to release a patch as soon as possible given the complexity of the problem. + +## Public Discussions + +Please avoid publicly discussing a potential security vulnerability. + +Let's take this offline and find a solution first, this limits the potential impact as much as possible. + +We appreciate your help! diff --git a/vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md b/vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md new file mode 100644 index 000000000..b5039e49c --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md @@ -0,0 +1,137 @@ +# `jwt-go` Version History + +The following version history is kept for historic purposes. To retrieve the current changes of each version, please refer to the change-log of the specific release versions on https://github.com/golang-jwt/jwt/releases. + +## 4.0.0 + +* Introduces support for Go modules. The `v4` version will be backwards compatible with `v3.x.y`. + +## 3.2.2 + +* Starting from this release, we are adopting the policy to support the most 2 recent versions of Go currently available. By the time of this release, this is Go 1.15 and 1.16 ([#28](https://github.com/golang-jwt/jwt/pull/28)). +* Fixed a potential issue that could occur when the verification of `exp`, `iat` or `nbf` was not required and contained invalid contents, i.e. non-numeric/date. Thanks for @thaJeztah for making us aware of that and @giorgos-f3 for originally reporting it to the formtech fork ([#40](https://github.com/golang-jwt/jwt/pull/40)). +* Added support for EdDSA / ED25519 ([#36](https://github.com/golang-jwt/jwt/pull/36)). +* Optimized allocations ([#33](https://github.com/golang-jwt/jwt/pull/33)). + +## 3.2.1 + +* **Import Path Change**: See MIGRATION_GUIDE.md for tips on updating your code + * Changed the import path from `github.com/dgrijalva/jwt-go` to `github.com/golang-jwt/jwt` +* Fixed type confusing issue between `string` and `[]string` in `VerifyAudience` ([#12](https://github.com/golang-jwt/jwt/pull/12)). This fixes CVE-2020-26160 + +#### 3.2.0 + +* Added method `ParseUnverified` to allow users to split up the tasks of parsing and validation +* HMAC signing method returns `ErrInvalidKeyType` instead of `ErrInvalidKey` where appropriate +* Added options to `request.ParseFromRequest`, which allows for an arbitrary list of modifiers to parsing behavior. Initial set include `WithClaims` and `WithParser`. Existing usage of this function will continue to work as before. +* Deprecated `ParseFromRequestWithClaims` to simplify API in the future. + +#### 3.1.0 + +* Improvements to `jwt` command line tool +* Added `SkipClaimsValidation` option to `Parser` +* Documentation updates + +#### 3.0.0 + +* **Compatibility Breaking Changes**: See MIGRATION_GUIDE.md for tips on updating your code + * Dropped support for `[]byte` keys when using RSA signing methods. This convenience feature could contribute to security vulnerabilities involving mismatched key types with signing methods. + * `ParseFromRequest` has been moved to `request` subpackage and usage has changed + * The `Claims` property on `Token` is now type `Claims` instead of `map[string]interface{}`. The default value is type `MapClaims`, which is an alias to `map[string]interface{}`. This makes it possible to use a custom type when decoding claims. +* Other Additions and Changes + * Added `Claims` interface type to allow users to decode the claims into a custom type + * Added `ParseWithClaims`, which takes a third argument of type `Claims`. Use this function instead of `Parse` if you have a custom type you'd like to decode into. + * Dramatically improved the functionality and flexibility of `ParseFromRequest`, which is now in the `request` subpackage + * Added `ParseFromRequestWithClaims` which is the `FromRequest` equivalent of `ParseWithClaims` + * Added new interface type `Extractor`, which is used for extracting JWT strings from http requests. Used with `ParseFromRequest` and `ParseFromRequestWithClaims`. + * Added several new, more specific, validation errors to error type bitmask + * Moved examples from README to executable example files + * Signing method registry is now thread safe + * Added new property to `ValidationError`, which contains the raw error returned by calls made by parse/verify (such as those returned by keyfunc or json parser) + +#### 2.7.0 + +This will likely be the last backwards compatible release before 3.0.0, excluding essential bug fixes. + +* Added new option `-show` to the `jwt` command that will just output the decoded token without verifying +* Error text for expired tokens includes how long it's been expired +* Fixed incorrect error returned from `ParseRSAPublicKeyFromPEM` +* Documentation updates + +#### 2.6.0 + +* Exposed inner error within ValidationError +* Fixed validation errors when using UseJSONNumber flag +* Added several unit tests + +#### 2.5.0 + +* Added support for signing method none. You shouldn't use this. The API tries to make this clear. +* Updated/fixed some documentation +* Added more helpful error message when trying to parse tokens that begin with `BEARER ` + +#### 2.4.0 + +* Added new type, Parser, to allow for configuration of various parsing parameters + * You can now specify a list of valid signing methods. Anything outside this set will be rejected. + * You can now opt to use the `json.Number` type instead of `float64` when parsing token JSON +* Added support for [Travis CI](https://travis-ci.org/dgrijalva/jwt-go) +* Fixed some bugs with ECDSA parsing + +#### 2.3.0 + +* Added support for ECDSA signing methods +* Added support for RSA PSS signing methods (requires go v1.4) + +#### 2.2.0 + +* Gracefully handle a `nil` `Keyfunc` being passed to `Parse`. Result will now be the parsed token and an error, instead of a panic. + +#### 2.1.0 + +Backwards compatible API change that was missed in 2.0.0. + +* The `SignedString` method on `Token` now takes `interface{}` instead of `[]byte` + +#### 2.0.0 + +There were two major reasons for breaking backwards compatibility with this update. The first was a refactor required to expand the width of the RSA and HMAC-SHA signing implementations. There will likely be no required code changes to support this change. + +The second update, while unfortunately requiring a small change in integration, is required to open up this library to other signing methods. Not all keys used for all signing methods have a single standard on-disk representation. Requiring `[]byte` as the type for all keys proved too limiting. Additionally, this implementation allows for pre-parsed tokens to be reused, which might matter in an application that parses a high volume of tokens with a small set of keys. Backwards compatibilty has been maintained for passing `[]byte` to the RSA signing methods, but they will also accept `*rsa.PublicKey` and `*rsa.PrivateKey`. + +It is likely the only integration change required here will be to change `func(t *jwt.Token) ([]byte, error)` to `func(t *jwt.Token) (interface{}, error)` when calling `Parse`. + +* **Compatibility Breaking Changes** + * `SigningMethodHS256` is now `*SigningMethodHMAC` instead of `type struct` + * `SigningMethodRS256` is now `*SigningMethodRSA` instead of `type struct` + * `KeyFunc` now returns `interface{}` instead of `[]byte` + * `SigningMethod.Sign` now takes `interface{}` instead of `[]byte` for the key + * `SigningMethod.Verify` now takes `interface{}` instead of `[]byte` for the key +* Renamed type `SigningMethodHS256` to `SigningMethodHMAC`. Specific sizes are now just instances of this type. + * Added public package global `SigningMethodHS256` + * Added public package global `SigningMethodHS384` + * Added public package global `SigningMethodHS512` +* Renamed type `SigningMethodRS256` to `SigningMethodRSA`. Specific sizes are now just instances of this type. + * Added public package global `SigningMethodRS256` + * Added public package global `SigningMethodRS384` + * Added public package global `SigningMethodRS512` +* Moved sample private key for HMAC tests from an inline value to a file on disk. Value is unchanged. +* Refactored the RSA implementation to be easier to read +* Exposed helper methods `ParseRSAPrivateKeyFromPEM` and `ParseRSAPublicKeyFromPEM` + +## 1.0.2 + +* Fixed bug in parsing public keys from certificates +* Added more tests around the parsing of keys for RS256 +* Code refactoring in RS256 implementation. No functional changes + +## 1.0.1 + +* Fixed panic if RS256 signing method was passed an invalid key + +## 1.0.0 + +* First versioned release +* API stabilized +* Supports creating, signing, parsing, and validating JWT tokens +* Supports RS256 and HS256 signing methods diff --git a/vendor/github.com/golang-jwt/jwt/v5/claims.go b/vendor/github.com/golang-jwt/jwt/v5/claims.go new file mode 100644 index 000000000..d50ff3dad --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/claims.go @@ -0,0 +1,16 @@ +package jwt + +// Claims represent any form of a JWT Claims Set according to +// https://datatracker.ietf.org/doc/html/rfc7519#section-4. In order to have a +// common basis for validation, it is required that an implementation is able to +// supply at least the claim names provided in +// https://datatracker.ietf.org/doc/html/rfc7519#section-4.1 namely `exp`, +// `iat`, `nbf`, `iss`, `sub` and `aud`. +type Claims interface { + GetExpirationTime() (*NumericDate, error) + GetIssuedAt() (*NumericDate, error) + GetNotBefore() (*NumericDate, error) + GetIssuer() (string, error) + GetSubject() (string, error) + GetAudience() (ClaimStrings, error) +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/doc.go b/vendor/github.com/golang-jwt/jwt/v5/doc.go new file mode 100644 index 000000000..a86dc1a3b --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/doc.go @@ -0,0 +1,4 @@ +// Package jwt is a Go implementation of JSON Web Tokens: http://self-issued.info/docs/draft-jones-json-web-token.html +// +// See README.md for more info. +package jwt diff --git a/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go b/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go new file mode 100644 index 000000000..c929e4a02 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go @@ -0,0 +1,134 @@ +package jwt + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rand" + "errors" + "math/big" +) + +var ( + // Sadly this is missing from crypto/ecdsa compared to crypto/rsa + ErrECDSAVerification = errors.New("crypto/ecdsa: verification error") +) + +// SigningMethodECDSA implements the ECDSA family of signing methods. +// Expects *ecdsa.PrivateKey for signing and *ecdsa.PublicKey for verification +type SigningMethodECDSA struct { + Name string + Hash crypto.Hash + KeySize int + CurveBits int +} + +// Specific instances for EC256 and company +var ( + SigningMethodES256 *SigningMethodECDSA + SigningMethodES384 *SigningMethodECDSA + SigningMethodES512 *SigningMethodECDSA +) + +func init() { + // ES256 + SigningMethodES256 = &SigningMethodECDSA{"ES256", crypto.SHA256, 32, 256} + RegisterSigningMethod(SigningMethodES256.Alg(), func() SigningMethod { + return SigningMethodES256 + }) + + // ES384 + SigningMethodES384 = &SigningMethodECDSA{"ES384", crypto.SHA384, 48, 384} + RegisterSigningMethod(SigningMethodES384.Alg(), func() SigningMethod { + return SigningMethodES384 + }) + + // ES512 + SigningMethodES512 = &SigningMethodECDSA{"ES512", crypto.SHA512, 66, 521} + RegisterSigningMethod(SigningMethodES512.Alg(), func() SigningMethod { + return SigningMethodES512 + }) +} + +func (m *SigningMethodECDSA) Alg() string { + return m.Name +} + +// Verify implements token verification for the SigningMethod. +// For this verify method, key must be an ecdsa.PublicKey struct +func (m *SigningMethodECDSA) Verify(signingString string, sig []byte, key interface{}) error { + // Get the key + var ecdsaKey *ecdsa.PublicKey + switch k := key.(type) { + case *ecdsa.PublicKey: + ecdsaKey = k + default: + return newError("ECDSA verify expects *ecdsa.PublicKey", ErrInvalidKeyType) + } + + if len(sig) != 2*m.KeySize { + return ErrECDSAVerification + } + + r := big.NewInt(0).SetBytes(sig[:m.KeySize]) + s := big.NewInt(0).SetBytes(sig[m.KeySize:]) + + // Create hasher + if !m.Hash.Available() { + return ErrHashUnavailable + } + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Verify the signature + if verifystatus := ecdsa.Verify(ecdsaKey, hasher.Sum(nil), r, s); verifystatus { + return nil + } + + return ErrECDSAVerification +} + +// Sign implements token signing for the SigningMethod. +// For this signing method, key must be an ecdsa.PrivateKey struct +func (m *SigningMethodECDSA) Sign(signingString string, key interface{}) ([]byte, error) { + // Get the key + var ecdsaKey *ecdsa.PrivateKey + switch k := key.(type) { + case *ecdsa.PrivateKey: + ecdsaKey = k + default: + return nil, newError("ECDSA sign expects *ecdsa.PrivateKey", ErrInvalidKeyType) + } + + // Create the hasher + if !m.Hash.Available() { + return nil, ErrHashUnavailable + } + + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Sign the string and return r, s + if r, s, err := ecdsa.Sign(rand.Reader, ecdsaKey, hasher.Sum(nil)); err == nil { + curveBits := ecdsaKey.Curve.Params().BitSize + + if m.CurveBits != curveBits { + return nil, ErrInvalidKey + } + + keyBytes := curveBits / 8 + if curveBits%8 > 0 { + keyBytes += 1 + } + + // We serialize the outputs (r and s) into big-endian byte arrays + // padded with zeros on the left to make sure the sizes work out. + // Output must be 2*keyBytes long. + out := make([]byte, 2*keyBytes) + r.FillBytes(out[0:keyBytes]) // r is assigned to the first half of output. + s.FillBytes(out[keyBytes:]) // s is assigned to the second half of output. + + return out, nil + } else { + return nil, err + } +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go b/vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go new file mode 100644 index 000000000..5700636d3 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go @@ -0,0 +1,69 @@ +package jwt + +import ( + "crypto/ecdsa" + "crypto/x509" + "encoding/pem" + "errors" +) + +var ( + ErrNotECPublicKey = errors.New("key is not a valid ECDSA public key") + ErrNotECPrivateKey = errors.New("key is not a valid ECDSA private key") +) + +// ParseECPrivateKeyFromPEM parses a PEM encoded Elliptic Curve Private Key Structure +func ParseECPrivateKeyFromPEM(key []byte) (*ecdsa.PrivateKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParseECPrivateKey(block.Bytes); err != nil { + if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { + return nil, err + } + } + + var pkey *ecdsa.PrivateKey + var ok bool + if pkey, ok = parsedKey.(*ecdsa.PrivateKey); !ok { + return nil, ErrNotECPrivateKey + } + + return pkey, nil +} + +// ParseECPublicKeyFromPEM parses a PEM encoded PKCS1 or PKCS8 public key +func ParseECPublicKeyFromPEM(key []byte) (*ecdsa.PublicKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { + if cert, err := x509.ParseCertificate(block.Bytes); err == nil { + parsedKey = cert.PublicKey + } else { + return nil, err + } + } + + var pkey *ecdsa.PublicKey + var ok bool + if pkey, ok = parsedKey.(*ecdsa.PublicKey); !ok { + return nil, ErrNotECPublicKey + } + + return pkey, nil +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/ed25519.go b/vendor/github.com/golang-jwt/jwt/v5/ed25519.go new file mode 100644 index 000000000..c2138119e --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/ed25519.go @@ -0,0 +1,79 @@ +package jwt + +import ( + "crypto" + "crypto/ed25519" + "crypto/rand" + "errors" +) + +var ( + ErrEd25519Verification = errors.New("ed25519: verification error") +) + +// SigningMethodEd25519 implements the EdDSA family. +// Expects ed25519.PrivateKey for signing and ed25519.PublicKey for verification +type SigningMethodEd25519 struct{} + +// Specific instance for EdDSA +var ( + SigningMethodEdDSA *SigningMethodEd25519 +) + +func init() { + SigningMethodEdDSA = &SigningMethodEd25519{} + RegisterSigningMethod(SigningMethodEdDSA.Alg(), func() SigningMethod { + return SigningMethodEdDSA + }) +} + +func (m *SigningMethodEd25519) Alg() string { + return "EdDSA" +} + +// Verify implements token verification for the SigningMethod. +// For this verify method, key must be an ed25519.PublicKey +func (m *SigningMethodEd25519) Verify(signingString string, sig []byte, key interface{}) error { + var ed25519Key ed25519.PublicKey + var ok bool + + if ed25519Key, ok = key.(ed25519.PublicKey); !ok { + return newError("Ed25519 verify expects ed25519.PublicKey", ErrInvalidKeyType) + } + + if len(ed25519Key) != ed25519.PublicKeySize { + return ErrInvalidKey + } + + // Verify the signature + if !ed25519.Verify(ed25519Key, []byte(signingString), sig) { + return ErrEd25519Verification + } + + return nil +} + +// Sign implements token signing for the SigningMethod. +// For this signing method, key must be an ed25519.PrivateKey +func (m *SigningMethodEd25519) Sign(signingString string, key interface{}) ([]byte, error) { + var ed25519Key crypto.Signer + var ok bool + + if ed25519Key, ok = key.(crypto.Signer); !ok { + return nil, newError("Ed25519 sign expects crypto.Signer", ErrInvalidKeyType) + } + + if _, ok := ed25519Key.Public().(ed25519.PublicKey); !ok { + return nil, ErrInvalidKey + } + + // Sign the string and return the result. ed25519 performs a two-pass hash + // as part of its algorithm. Therefore, we need to pass a non-prehashed + // message into the Sign function, as indicated by crypto.Hash(0) + sig, err := ed25519Key.Sign(rand.Reader, []byte(signingString), crypto.Hash(0)) + if err != nil { + return nil, err + } + + return sig, nil +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go b/vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go new file mode 100644 index 000000000..cdb5e68e8 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go @@ -0,0 +1,64 @@ +package jwt + +import ( + "crypto" + "crypto/ed25519" + "crypto/x509" + "encoding/pem" + "errors" +) + +var ( + ErrNotEdPrivateKey = errors.New("key is not a valid Ed25519 private key") + ErrNotEdPublicKey = errors.New("key is not a valid Ed25519 public key") +) + +// ParseEdPrivateKeyFromPEM parses a PEM-encoded Edwards curve private key +func ParseEdPrivateKeyFromPEM(key []byte) (crypto.PrivateKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { + return nil, err + } + + var pkey ed25519.PrivateKey + var ok bool + if pkey, ok = parsedKey.(ed25519.PrivateKey); !ok { + return nil, ErrNotEdPrivateKey + } + + return pkey, nil +} + +// ParseEdPublicKeyFromPEM parses a PEM-encoded Edwards curve public key +func ParseEdPublicKeyFromPEM(key []byte) (crypto.PublicKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { + return nil, err + } + + var pkey ed25519.PublicKey + var ok bool + if pkey, ok = parsedKey.(ed25519.PublicKey); !ok { + return nil, ErrNotEdPublicKey + } + + return pkey, nil +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors.go b/vendor/github.com/golang-jwt/jwt/v5/errors.go new file mode 100644 index 000000000..23bb616dd --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/errors.go @@ -0,0 +1,49 @@ +package jwt + +import ( + "errors" + "strings" +) + +var ( + ErrInvalidKey = errors.New("key is invalid") + ErrInvalidKeyType = errors.New("key is of invalid type") + ErrHashUnavailable = errors.New("the requested hash function is unavailable") + ErrTokenMalformed = errors.New("token is malformed") + ErrTokenUnverifiable = errors.New("token is unverifiable") + ErrTokenSignatureInvalid = errors.New("token signature is invalid") + ErrTokenRequiredClaimMissing = errors.New("token is missing required claim") + ErrTokenInvalidAudience = errors.New("token has invalid audience") + ErrTokenExpired = errors.New("token is expired") + ErrTokenUsedBeforeIssued = errors.New("token used before issued") + ErrTokenInvalidIssuer = errors.New("token has invalid issuer") + ErrTokenInvalidSubject = errors.New("token has invalid subject") + ErrTokenNotValidYet = errors.New("token is not valid yet") + ErrTokenInvalidId = errors.New("token has invalid id") + ErrTokenInvalidClaims = errors.New("token has invalid claims") + ErrInvalidType = errors.New("invalid type for claim") +) + +// joinedError is an error type that works similar to what [errors.Join] +// produces, with the exception that it has a nice error string; mainly its +// error messages are concatenated using a comma, rather than a newline. +type joinedError struct { + errs []error +} + +func (je joinedError) Error() string { + msg := []string{} + for _, err := range je.errs { + msg = append(msg, err.Error()) + } + + return strings.Join(msg, ", ") +} + +// joinErrors joins together multiple errors. Useful for scenarios where +// multiple errors next to each other occur, e.g., in claims validation. +func joinErrors(errs ...error) error { + return &joinedError{ + errs: errs, + } +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go b/vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go new file mode 100644 index 000000000..a893d355e --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go @@ -0,0 +1,47 @@ +//go:build go1.20 +// +build go1.20 + +package jwt + +import ( + "fmt" +) + +// Unwrap implements the multiple error unwrapping for this error type, which is +// possible in Go 1.20. +func (je joinedError) Unwrap() []error { + return je.errs +} + +// newError creates a new error message with a detailed error message. The +// message will be prefixed with the contents of the supplied error type. +// Additionally, more errors, that provide more context can be supplied which +// will be appended to the message. This makes use of Go 1.20's possibility to +// include more than one %w formatting directive in [fmt.Errorf]. +// +// For example, +// +// newError("no keyfunc was provided", ErrTokenUnverifiable) +// +// will produce the error string +// +// "token is unverifiable: no keyfunc was provided" +func newError(message string, err error, more ...error) error { + var format string + var args []any + if message != "" { + format = "%w: %s" + args = []any{err, message} + } else { + format = "%w" + args = []any{err} + } + + for _, e := range more { + format += ": %w" + args = append(args, e) + } + + err = fmt.Errorf(format, args...) + return err +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go b/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go new file mode 100644 index 000000000..2ad542f00 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go @@ -0,0 +1,78 @@ +//go:build !go1.20 +// +build !go1.20 + +package jwt + +import ( + "errors" + "fmt" +) + +// Is implements checking for multiple errors using [errors.Is], since multiple +// error unwrapping is not possible in versions less than Go 1.20. +func (je joinedError) Is(err error) bool { + for _, e := range je.errs { + if errors.Is(e, err) { + return true + } + } + + return false +} + +// wrappedErrors is a workaround for wrapping multiple errors in environments +// where Go 1.20 is not available. It basically uses the already implemented +// functionality of joinedError to handle multiple errors with supplies a +// custom error message that is identical to the one we produce in Go 1.20 using +// multiple %w directives. +type wrappedErrors struct { + msg string + joinedError +} + +// Error returns the stored error string +func (we wrappedErrors) Error() string { + return we.msg +} + +// newError creates a new error message with a detailed error message. The +// message will be prefixed with the contents of the supplied error type. +// Additionally, more errors, that provide more context can be supplied which +// will be appended to the message. Since we cannot use of Go 1.20's possibility +// to include more than one %w formatting directive in [fmt.Errorf], we have to +// emulate that. +// +// For example, +// +// newError("no keyfunc was provided", ErrTokenUnverifiable) +// +// will produce the error string +// +// "token is unverifiable: no keyfunc was provided" +func newError(message string, err error, more ...error) error { + // We cannot wrap multiple errors here with %w, so we have to be a little + // bit creative. Basically, we are using %s instead of %w to produce the + // same error message and then throw the result into a custom error struct. + var format string + var args []any + if message != "" { + format = "%s: %s" + args = []any{err, message} + } else { + format = "%s" + args = []any{err} + } + errs := []error{err} + + for _, e := range more { + format += ": %s" + args = append(args, e) + errs = append(errs, e) + } + + err = &wrappedErrors{ + msg: fmt.Sprintf(format, args...), + joinedError: joinedError{errs: errs}, + } + return err +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/hmac.go b/vendor/github.com/golang-jwt/jwt/v5/hmac.go new file mode 100644 index 000000000..aca600ce1 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/hmac.go @@ -0,0 +1,104 @@ +package jwt + +import ( + "crypto" + "crypto/hmac" + "errors" +) + +// SigningMethodHMAC implements the HMAC-SHA family of signing methods. +// Expects key type of []byte for both signing and validation +type SigningMethodHMAC struct { + Name string + Hash crypto.Hash +} + +// Specific instances for HS256 and company +var ( + SigningMethodHS256 *SigningMethodHMAC + SigningMethodHS384 *SigningMethodHMAC + SigningMethodHS512 *SigningMethodHMAC + ErrSignatureInvalid = errors.New("signature is invalid") +) + +func init() { + // HS256 + SigningMethodHS256 = &SigningMethodHMAC{"HS256", crypto.SHA256} + RegisterSigningMethod(SigningMethodHS256.Alg(), func() SigningMethod { + return SigningMethodHS256 + }) + + // HS384 + SigningMethodHS384 = &SigningMethodHMAC{"HS384", crypto.SHA384} + RegisterSigningMethod(SigningMethodHS384.Alg(), func() SigningMethod { + return SigningMethodHS384 + }) + + // HS512 + SigningMethodHS512 = &SigningMethodHMAC{"HS512", crypto.SHA512} + RegisterSigningMethod(SigningMethodHS512.Alg(), func() SigningMethod { + return SigningMethodHS512 + }) +} + +func (m *SigningMethodHMAC) Alg() string { + return m.Name +} + +// Verify implements token verification for the SigningMethod. Returns nil if +// the signature is valid. Key must be []byte. +// +// Note it is not advised to provide a []byte which was converted from a 'human +// readable' string using a subset of ASCII characters. To maximize entropy, you +// should ideally be providing a []byte key which was produced from a +// cryptographically random source, e.g. crypto/rand. Additional information +// about this, and why we intentionally are not supporting string as a key can +// be found on our usage guide +// https://golang-jwt.github.io/jwt/usage/signing_methods/#signing-methods-and-key-types. +func (m *SigningMethodHMAC) Verify(signingString string, sig []byte, key interface{}) error { + // Verify the key is the right type + keyBytes, ok := key.([]byte) + if !ok { + return newError("HMAC verify expects []byte", ErrInvalidKeyType) + } + + // Can we use the specified hashing method? + if !m.Hash.Available() { + return ErrHashUnavailable + } + + // This signing method is symmetric, so we validate the signature + // by reproducing the signature from the signing string and key, then + // comparing that against the provided signature. + hasher := hmac.New(m.Hash.New, keyBytes) + hasher.Write([]byte(signingString)) + if !hmac.Equal(sig, hasher.Sum(nil)) { + return ErrSignatureInvalid + } + + // No validation errors. Signature is good. + return nil +} + +// Sign implements token signing for the SigningMethod. Key must be []byte. +// +// Note it is not advised to provide a []byte which was converted from a 'human +// readable' string using a subset of ASCII characters. To maximize entropy, you +// should ideally be providing a []byte key which was produced from a +// cryptographically random source, e.g. crypto/rand. Additional information +// about this, and why we intentionally are not supporting string as a key can +// be found on our usage guide https://golang-jwt.github.io/jwt/usage/signing_methods/. +func (m *SigningMethodHMAC) Sign(signingString string, key interface{}) ([]byte, error) { + if keyBytes, ok := key.([]byte); ok { + if !m.Hash.Available() { + return nil, ErrHashUnavailable + } + + hasher := hmac.New(m.Hash.New, keyBytes) + hasher.Write([]byte(signingString)) + + return hasher.Sum(nil), nil + } + + return nil, newError("HMAC sign expects []byte", ErrInvalidKeyType) +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/map_claims.go b/vendor/github.com/golang-jwt/jwt/v5/map_claims.go new file mode 100644 index 000000000..b2b51a1f8 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/map_claims.go @@ -0,0 +1,109 @@ +package jwt + +import ( + "encoding/json" + "fmt" +) + +// MapClaims is a claims type that uses the map[string]interface{} for JSON +// decoding. This is the default claims type if you don't supply one +type MapClaims map[string]interface{} + +// GetExpirationTime implements the Claims interface. +func (m MapClaims) GetExpirationTime() (*NumericDate, error) { + return m.parseNumericDate("exp") +} + +// GetNotBefore implements the Claims interface. +func (m MapClaims) GetNotBefore() (*NumericDate, error) { + return m.parseNumericDate("nbf") +} + +// GetIssuedAt implements the Claims interface. +func (m MapClaims) GetIssuedAt() (*NumericDate, error) { + return m.parseNumericDate("iat") +} + +// GetAudience implements the Claims interface. +func (m MapClaims) GetAudience() (ClaimStrings, error) { + return m.parseClaimsString("aud") +} + +// GetIssuer implements the Claims interface. +func (m MapClaims) GetIssuer() (string, error) { + return m.parseString("iss") +} + +// GetSubject implements the Claims interface. +func (m MapClaims) GetSubject() (string, error) { + return m.parseString("sub") +} + +// parseNumericDate tries to parse a key in the map claims type as a number +// date. This will succeed, if the underlying type is either a [float64] or a +// [json.Number]. Otherwise, nil will be returned. +func (m MapClaims) parseNumericDate(key string) (*NumericDate, error) { + v, ok := m[key] + if !ok { + return nil, nil + } + + switch exp := v.(type) { + case float64: + if exp == 0 { + return nil, nil + } + + return newNumericDateFromSeconds(exp), nil + case json.Number: + v, _ := exp.Float64() + + return newNumericDateFromSeconds(v), nil + } + + return nil, newError(fmt.Sprintf("%s is invalid", key), ErrInvalidType) +} + +// parseClaimsString tries to parse a key in the map claims type as a +// [ClaimsStrings] type, which can either be a string or an array of string. +func (m MapClaims) parseClaimsString(key string) (ClaimStrings, error) { + var cs []string + switch v := m[key].(type) { + case string: + cs = append(cs, v) + case []string: + cs = v + case []interface{}: + for _, a := range v { + vs, ok := a.(string) + if !ok { + return nil, newError(fmt.Sprintf("%s is invalid", key), ErrInvalidType) + } + cs = append(cs, vs) + } + } + + return cs, nil +} + +// parseString tries to parse a key in the map claims type as a [string] type. +// If the key does not exist, an empty string is returned. If the key has the +// wrong type, an error is returned. +func (m MapClaims) parseString(key string) (string, error) { + var ( + ok bool + raw interface{} + iss string + ) + raw, ok = m[key] + if !ok { + return "", nil + } + + iss, ok = raw.(string) + if !ok { + return "", newError(fmt.Sprintf("%s is invalid", key), ErrInvalidType) + } + + return iss, nil +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/none.go b/vendor/github.com/golang-jwt/jwt/v5/none.go new file mode 100644 index 000000000..685c2ea30 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/none.go @@ -0,0 +1,50 @@ +package jwt + +// SigningMethodNone implements the none signing method. This is required by the spec +// but you probably should never use it. +var SigningMethodNone *signingMethodNone + +const UnsafeAllowNoneSignatureType unsafeNoneMagicConstant = "none signing method allowed" + +var NoneSignatureTypeDisallowedError error + +type signingMethodNone struct{} +type unsafeNoneMagicConstant string + +func init() { + SigningMethodNone = &signingMethodNone{} + NoneSignatureTypeDisallowedError = newError("'none' signature type is not allowed", ErrTokenUnverifiable) + + RegisterSigningMethod(SigningMethodNone.Alg(), func() SigningMethod { + return SigningMethodNone + }) +} + +func (m *signingMethodNone) Alg() string { + return "none" +} + +// Only allow 'none' alg type if UnsafeAllowNoneSignatureType is specified as the key +func (m *signingMethodNone) Verify(signingString string, sig []byte, key interface{}) (err error) { + // Key must be UnsafeAllowNoneSignatureType to prevent accidentally + // accepting 'none' signing method + if _, ok := key.(unsafeNoneMagicConstant); !ok { + return NoneSignatureTypeDisallowedError + } + // If signing method is none, signature must be an empty string + if len(sig) != 0 { + return newError("'none' signing method with non-empty signature", ErrTokenUnverifiable) + } + + // Accept 'none' signing method. + return nil +} + +// Only allow 'none' signing if UnsafeAllowNoneSignatureType is specified as the key +func (m *signingMethodNone) Sign(signingString string, key interface{}) ([]byte, error) { + if _, ok := key.(unsafeNoneMagicConstant); ok { + return []byte{}, nil + } + + return nil, NoneSignatureTypeDisallowedError +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/parser.go b/vendor/github.com/golang-jwt/jwt/v5/parser.go new file mode 100644 index 000000000..ecf99af78 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/parser.go @@ -0,0 +1,238 @@ +package jwt + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "strings" +) + +type Parser struct { + // If populated, only these methods will be considered valid. + validMethods []string + + // Use JSON Number format in JSON decoder. + useJSONNumber bool + + // Skip claims validation during token parsing. + skipClaimsValidation bool + + validator *Validator + + decodeStrict bool + + decodePaddingAllowed bool +} + +// NewParser creates a new Parser with the specified options +func NewParser(options ...ParserOption) *Parser { + p := &Parser{ + validator: &Validator{}, + } + + // Loop through our parsing options and apply them + for _, option := range options { + option(p) + } + + return p +} + +// Parse parses, validates, verifies the signature and returns the parsed token. +// keyFunc will receive the parsed token and should return the key for validating. +func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { + return p.ParseWithClaims(tokenString, MapClaims{}, keyFunc) +} + +// ParseWithClaims parses, validates, and verifies like Parse, but supplies a default object implementing the Claims +// interface. This provides default values which can be overridden and allows a caller to use their own type, rather +// than the default MapClaims implementation of Claims. +// +// Note: If you provide a custom claim implementation that embeds one of the standard claims (such as RegisteredClaims), +// make sure that a) you either embed a non-pointer version of the claims or b) if you are using a pointer, allocate the +// proper memory for it before passing in the overall claims, otherwise you might run into a panic. +func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) { + token, parts, err := p.ParseUnverified(tokenString, claims) + if err != nil { + return token, err + } + + // Verify signing method is in the required set + if p.validMethods != nil { + var signingMethodValid = false + var alg = token.Method.Alg() + for _, m := range p.validMethods { + if m == alg { + signingMethodValid = true + break + } + } + if !signingMethodValid { + // signing method is not in the listed set + return token, newError(fmt.Sprintf("signing method %v is invalid", alg), ErrTokenSignatureInvalid) + } + } + + // Decode signature + token.Signature, err = p.DecodeSegment(parts[2]) + if err != nil { + return token, newError("could not base64 decode signature", ErrTokenMalformed, err) + } + text := strings.Join(parts[0:2], ".") + + // Lookup key(s) + if keyFunc == nil { + // keyFunc was not provided. short circuiting validation + return token, newError("no keyfunc was provided", ErrTokenUnverifiable) + } + + got, err := keyFunc(token) + if err != nil { + return token, newError("error while executing keyfunc", ErrTokenUnverifiable, err) + } + + switch have := got.(type) { + case VerificationKeySet: + if len(have.Keys) == 0 { + return token, newError("keyfunc returned empty verification key set", ErrTokenUnverifiable) + } + // Iterate through keys and verify signature, skipping the rest when a match is found. + // Return the last error if no match is found. + for _, key := range have.Keys { + if err = token.Method.Verify(text, token.Signature, key); err == nil { + break + } + } + default: + err = token.Method.Verify(text, token.Signature, have) + } + if err != nil { + return token, newError("", ErrTokenSignatureInvalid, err) + } + + // Validate Claims + if !p.skipClaimsValidation { + // Make sure we have at least a default validator + if p.validator == nil { + p.validator = NewValidator() + } + + if err := p.validator.Validate(claims); err != nil { + return token, newError("", ErrTokenInvalidClaims, err) + } + } + + // No errors so far, token is valid. + token.Valid = true + + return token, nil +} + +// ParseUnverified parses the token but doesn't validate the signature. +// +// WARNING: Don't use this method unless you know what you're doing. +// +// It's only ever useful in cases where you know the signature is valid (since it has already +// been or will be checked elsewhere in the stack) and you want to extract values from it. +func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { + parts = strings.Split(tokenString, ".") + if len(parts) != 3 { + return nil, parts, newError("token contains an invalid number of segments", ErrTokenMalformed) + } + + token = &Token{Raw: tokenString} + + // parse Header + var headerBytes []byte + if headerBytes, err = p.DecodeSegment(parts[0]); err != nil { + return token, parts, newError("could not base64 decode header", ErrTokenMalformed, err) + } + if err = json.Unmarshal(headerBytes, &token.Header); err != nil { + return token, parts, newError("could not JSON decode header", ErrTokenMalformed, err) + } + + // parse Claims + token.Claims = claims + + claimBytes, err := p.DecodeSegment(parts[1]) + if err != nil { + return token, parts, newError("could not base64 decode claim", ErrTokenMalformed, err) + } + + // If `useJSONNumber` is enabled then we must use *json.Decoder to decode + // the claims. However, this comes with a performance penalty so only use + // it if we must and, otherwise, simple use json.Unmarshal. + if !p.useJSONNumber { + // JSON Unmarshal. Special case for map type to avoid weird pointer behavior. + if c, ok := token.Claims.(MapClaims); ok { + err = json.Unmarshal(claimBytes, &c) + } else { + err = json.Unmarshal(claimBytes, &claims) + } + } else { + dec := json.NewDecoder(bytes.NewBuffer(claimBytes)) + dec.UseNumber() + // JSON Decode. Special case for map type to avoid weird pointer behavior. + if c, ok := token.Claims.(MapClaims); ok { + err = dec.Decode(&c) + } else { + err = dec.Decode(&claims) + } + } + if err != nil { + return token, parts, newError("could not JSON decode claim", ErrTokenMalformed, err) + } + + // Lookup signature method + if method, ok := token.Header["alg"].(string); ok { + if token.Method = GetSigningMethod(method); token.Method == nil { + return token, parts, newError("signing method (alg) is unavailable", ErrTokenUnverifiable) + } + } else { + return token, parts, newError("signing method (alg) is unspecified", ErrTokenUnverifiable) + } + + return token, parts, nil +} + +// DecodeSegment decodes a JWT specific base64url encoding. This function will +// take into account whether the [Parser] is configured with additional options, +// such as [WithStrictDecoding] or [WithPaddingAllowed]. +func (p *Parser) DecodeSegment(seg string) ([]byte, error) { + encoding := base64.RawURLEncoding + + if p.decodePaddingAllowed { + if l := len(seg) % 4; l > 0 { + seg += strings.Repeat("=", 4-l) + } + encoding = base64.URLEncoding + } + + if p.decodeStrict { + encoding = encoding.Strict() + } + return encoding.DecodeString(seg) +} + +// Parse parses, validates, verifies the signature and returns the parsed token. +// keyFunc will receive the parsed token and should return the cryptographic key +// for verifying the signature. The caller is strongly encouraged to set the +// WithValidMethods option to validate the 'alg' claim in the token matches the +// expected algorithm. For more details about the importance of validating the +// 'alg' claim, see +// https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/ +func Parse(tokenString string, keyFunc Keyfunc, options ...ParserOption) (*Token, error) { + return NewParser(options...).Parse(tokenString, keyFunc) +} + +// ParseWithClaims is a shortcut for NewParser().ParseWithClaims(). +// +// Note: If you provide a custom claim implementation that embeds one of the +// standard claims (such as RegisteredClaims), make sure that a) you either +// embed a non-pointer version of the claims or b) if you are using a pointer, +// allocate the proper memory for it before passing in the overall claims, +// otherwise you might run into a panic. +func ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc, options ...ParserOption) (*Token, error) { + return NewParser(options...).ParseWithClaims(tokenString, claims, keyFunc) +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/parser_option.go b/vendor/github.com/golang-jwt/jwt/v5/parser_option.go new file mode 100644 index 000000000..88a780fbd --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/parser_option.go @@ -0,0 +1,128 @@ +package jwt + +import "time" + +// ParserOption is used to implement functional-style options that modify the +// behavior of the parser. To add new options, just create a function (ideally +// beginning with With or Without) that returns an anonymous function that takes +// a *Parser type as input and manipulates its configuration accordingly. +type ParserOption func(*Parser) + +// WithValidMethods is an option to supply algorithm methods that the parser +// will check. Only those methods will be considered valid. It is heavily +// encouraged to use this option in order to prevent attacks such as +// https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/. +func WithValidMethods(methods []string) ParserOption { + return func(p *Parser) { + p.validMethods = methods + } +} + +// WithJSONNumber is an option to configure the underlying JSON parser with +// UseNumber. +func WithJSONNumber() ParserOption { + return func(p *Parser) { + p.useJSONNumber = true + } +} + +// WithoutClaimsValidation is an option to disable claims validation. This +// option should only be used if you exactly know what you are doing. +func WithoutClaimsValidation() ParserOption { + return func(p *Parser) { + p.skipClaimsValidation = true + } +} + +// WithLeeway returns the ParserOption for specifying the leeway window. +func WithLeeway(leeway time.Duration) ParserOption { + return func(p *Parser) { + p.validator.leeway = leeway + } +} + +// WithTimeFunc returns the ParserOption for specifying the time func. The +// primary use-case for this is testing. If you are looking for a way to account +// for clock-skew, WithLeeway should be used instead. +func WithTimeFunc(f func() time.Time) ParserOption { + return func(p *Parser) { + p.validator.timeFunc = f + } +} + +// WithIssuedAt returns the ParserOption to enable verification +// of issued-at. +func WithIssuedAt() ParserOption { + return func(p *Parser) { + p.validator.verifyIat = true + } +} + +// WithExpirationRequired returns the ParserOption to make exp claim required. +// By default exp claim is optional. +func WithExpirationRequired() ParserOption { + return func(p *Parser) { + p.validator.requireExp = true + } +} + +// WithAudience configures the validator to require the specified audience in +// the `aud` claim. Validation will fail if the audience is not listed in the +// token or the `aud` claim is missing. +// +// NOTE: While the `aud` claim is OPTIONAL in a JWT, the handling of it is +// application-specific. Since this validation API is helping developers in +// writing secure application, we decided to REQUIRE the existence of the claim, +// if an audience is expected. +func WithAudience(aud string) ParserOption { + return func(p *Parser) { + p.validator.expectedAud = aud + } +} + +// WithIssuer configures the validator to require the specified issuer in the +// `iss` claim. Validation will fail if a different issuer is specified in the +// token or the `iss` claim is missing. +// +// NOTE: While the `iss` claim is OPTIONAL in a JWT, the handling of it is +// application-specific. Since this validation API is helping developers in +// writing secure application, we decided to REQUIRE the existence of the claim, +// if an issuer is expected. +func WithIssuer(iss string) ParserOption { + return func(p *Parser) { + p.validator.expectedIss = iss + } +} + +// WithSubject configures the validator to require the specified subject in the +// `sub` claim. Validation will fail if a different subject is specified in the +// token or the `sub` claim is missing. +// +// NOTE: While the `sub` claim is OPTIONAL in a JWT, the handling of it is +// application-specific. Since this validation API is helping developers in +// writing secure application, we decided to REQUIRE the existence of the claim, +// if a subject is expected. +func WithSubject(sub string) ParserOption { + return func(p *Parser) { + p.validator.expectedSub = sub + } +} + +// WithPaddingAllowed will enable the codec used for decoding JWTs to allow +// padding. Note that the JWS RFC7515 states that the tokens will utilize a +// Base64url encoding with no padding. Unfortunately, some implementations of +// JWT are producing non-standard tokens, and thus require support for decoding. +func WithPaddingAllowed() ParserOption { + return func(p *Parser) { + p.decodePaddingAllowed = true + } +} + +// WithStrictDecoding will switch the codec used for decoding JWTs into strict +// mode. In this mode, the decoder requires that trailing padding bits are zero, +// as described in RFC 4648 section 3.5. +func WithStrictDecoding() ParserOption { + return func(p *Parser) { + p.decodeStrict = true + } +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/registered_claims.go b/vendor/github.com/golang-jwt/jwt/v5/registered_claims.go new file mode 100644 index 000000000..77951a531 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/registered_claims.go @@ -0,0 +1,63 @@ +package jwt + +// RegisteredClaims are a structured version of the JWT Claims Set, +// restricted to Registered Claim Names, as referenced at +// https://datatracker.ietf.org/doc/html/rfc7519#section-4.1 +// +// This type can be used on its own, but then additional private and +// public claims embedded in the JWT will not be parsed. The typical use-case +// therefore is to embedded this in a user-defined claim type. +// +// See examples for how to use this with your own claim types. +type RegisteredClaims struct { + // the `iss` (Issuer) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.1 + Issuer string `json:"iss,omitempty"` + + // the `sub` (Subject) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.2 + Subject string `json:"sub,omitempty"` + + // the `aud` (Audience) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.3 + Audience ClaimStrings `json:"aud,omitempty"` + + // the `exp` (Expiration Time) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.4 + ExpiresAt *NumericDate `json:"exp,omitempty"` + + // the `nbf` (Not Before) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.5 + NotBefore *NumericDate `json:"nbf,omitempty"` + + // the `iat` (Issued At) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.6 + IssuedAt *NumericDate `json:"iat,omitempty"` + + // the `jti` (JWT ID) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.7 + ID string `json:"jti,omitempty"` +} + +// GetExpirationTime implements the Claims interface. +func (c RegisteredClaims) GetExpirationTime() (*NumericDate, error) { + return c.ExpiresAt, nil +} + +// GetNotBefore implements the Claims interface. +func (c RegisteredClaims) GetNotBefore() (*NumericDate, error) { + return c.NotBefore, nil +} + +// GetIssuedAt implements the Claims interface. +func (c RegisteredClaims) GetIssuedAt() (*NumericDate, error) { + return c.IssuedAt, nil +} + +// GetAudience implements the Claims interface. +func (c RegisteredClaims) GetAudience() (ClaimStrings, error) { + return c.Audience, nil +} + +// GetIssuer implements the Claims interface. +func (c RegisteredClaims) GetIssuer() (string, error) { + return c.Issuer, nil +} + +// GetSubject implements the Claims interface. +func (c RegisteredClaims) GetSubject() (string, error) { + return c.Subject, nil +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa.go b/vendor/github.com/golang-jwt/jwt/v5/rsa.go new file mode 100644 index 000000000..83cbee6ae --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/rsa.go @@ -0,0 +1,93 @@ +package jwt + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" +) + +// SigningMethodRSA implements the RSA family of signing methods. +// Expects *rsa.PrivateKey for signing and *rsa.PublicKey for validation +type SigningMethodRSA struct { + Name string + Hash crypto.Hash +} + +// Specific instances for RS256 and company +var ( + SigningMethodRS256 *SigningMethodRSA + SigningMethodRS384 *SigningMethodRSA + SigningMethodRS512 *SigningMethodRSA +) + +func init() { + // RS256 + SigningMethodRS256 = &SigningMethodRSA{"RS256", crypto.SHA256} + RegisterSigningMethod(SigningMethodRS256.Alg(), func() SigningMethod { + return SigningMethodRS256 + }) + + // RS384 + SigningMethodRS384 = &SigningMethodRSA{"RS384", crypto.SHA384} + RegisterSigningMethod(SigningMethodRS384.Alg(), func() SigningMethod { + return SigningMethodRS384 + }) + + // RS512 + SigningMethodRS512 = &SigningMethodRSA{"RS512", crypto.SHA512} + RegisterSigningMethod(SigningMethodRS512.Alg(), func() SigningMethod { + return SigningMethodRS512 + }) +} + +func (m *SigningMethodRSA) Alg() string { + return m.Name +} + +// Verify implements token verification for the SigningMethod +// For this signing method, must be an *rsa.PublicKey structure. +func (m *SigningMethodRSA) Verify(signingString string, sig []byte, key interface{}) error { + var rsaKey *rsa.PublicKey + var ok bool + + if rsaKey, ok = key.(*rsa.PublicKey); !ok { + return newError("RSA verify expects *rsa.PublicKey", ErrInvalidKeyType) + } + + // Create hasher + if !m.Hash.Available() { + return ErrHashUnavailable + } + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Verify the signature + return rsa.VerifyPKCS1v15(rsaKey, m.Hash, hasher.Sum(nil), sig) +} + +// Sign implements token signing for the SigningMethod +// For this signing method, must be an *rsa.PrivateKey structure. +func (m *SigningMethodRSA) Sign(signingString string, key interface{}) ([]byte, error) { + var rsaKey *rsa.PrivateKey + var ok bool + + // Validate type of key + if rsaKey, ok = key.(*rsa.PrivateKey); !ok { + return nil, newError("RSA sign expects *rsa.PrivateKey", ErrInvalidKeyType) + } + + // Create the hasher + if !m.Hash.Available() { + return nil, ErrHashUnavailable + } + + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Sign the string and return the encoded bytes + if sigBytes, err := rsa.SignPKCS1v15(rand.Reader, rsaKey, m.Hash, hasher.Sum(nil)); err == nil { + return sigBytes, nil + } else { + return nil, err + } +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go b/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go new file mode 100644 index 000000000..28c386ec4 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go @@ -0,0 +1,135 @@ +//go:build go1.4 +// +build go1.4 + +package jwt + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" +) + +// SigningMethodRSAPSS implements the RSAPSS family of signing methods signing methods +type SigningMethodRSAPSS struct { + *SigningMethodRSA + Options *rsa.PSSOptions + // VerifyOptions is optional. If set overrides Options for rsa.VerifyPPS. + // Used to accept tokens signed with rsa.PSSSaltLengthAuto, what doesn't follow + // https://tools.ietf.org/html/rfc7518#section-3.5 but was used previously. + // See https://github.com/dgrijalva/jwt-go/issues/285#issuecomment-437451244 for details. + VerifyOptions *rsa.PSSOptions +} + +// Specific instances for RS/PS and company. +var ( + SigningMethodPS256 *SigningMethodRSAPSS + SigningMethodPS384 *SigningMethodRSAPSS + SigningMethodPS512 *SigningMethodRSAPSS +) + +func init() { + // PS256 + SigningMethodPS256 = &SigningMethodRSAPSS{ + SigningMethodRSA: &SigningMethodRSA{ + Name: "PS256", + Hash: crypto.SHA256, + }, + Options: &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthEqualsHash, + }, + VerifyOptions: &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + }, + } + RegisterSigningMethod(SigningMethodPS256.Alg(), func() SigningMethod { + return SigningMethodPS256 + }) + + // PS384 + SigningMethodPS384 = &SigningMethodRSAPSS{ + SigningMethodRSA: &SigningMethodRSA{ + Name: "PS384", + Hash: crypto.SHA384, + }, + Options: &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthEqualsHash, + }, + VerifyOptions: &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + }, + } + RegisterSigningMethod(SigningMethodPS384.Alg(), func() SigningMethod { + return SigningMethodPS384 + }) + + // PS512 + SigningMethodPS512 = &SigningMethodRSAPSS{ + SigningMethodRSA: &SigningMethodRSA{ + Name: "PS512", + Hash: crypto.SHA512, + }, + Options: &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthEqualsHash, + }, + VerifyOptions: &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + }, + } + RegisterSigningMethod(SigningMethodPS512.Alg(), func() SigningMethod { + return SigningMethodPS512 + }) +} + +// Verify implements token verification for the SigningMethod. +// For this verify method, key must be an rsa.PublicKey struct +func (m *SigningMethodRSAPSS) Verify(signingString string, sig []byte, key interface{}) error { + var rsaKey *rsa.PublicKey + switch k := key.(type) { + case *rsa.PublicKey: + rsaKey = k + default: + return newError("RSA-PSS verify expects *rsa.PublicKey", ErrInvalidKeyType) + } + + // Create hasher + if !m.Hash.Available() { + return ErrHashUnavailable + } + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + opts := m.Options + if m.VerifyOptions != nil { + opts = m.VerifyOptions + } + + return rsa.VerifyPSS(rsaKey, m.Hash, hasher.Sum(nil), sig, opts) +} + +// Sign implements token signing for the SigningMethod. +// For this signing method, key must be an rsa.PrivateKey struct +func (m *SigningMethodRSAPSS) Sign(signingString string, key interface{}) ([]byte, error) { + var rsaKey *rsa.PrivateKey + + switch k := key.(type) { + case *rsa.PrivateKey: + rsaKey = k + default: + return nil, newError("RSA-PSS sign expects *rsa.PrivateKey", ErrInvalidKeyType) + } + + // Create the hasher + if !m.Hash.Available() { + return nil, ErrHashUnavailable + } + + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Sign the string and return the encoded bytes + if sigBytes, err := rsa.SignPSS(rand.Reader, rsaKey, m.Hash, hasher.Sum(nil), m.Options); err == nil { + return sigBytes, nil + } else { + return nil, err + } +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go b/vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go new file mode 100644 index 000000000..b3aeebbe1 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go @@ -0,0 +1,107 @@ +package jwt + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" +) + +var ( + ErrKeyMustBePEMEncoded = errors.New("invalid key: Key must be a PEM encoded PKCS1 or PKCS8 key") + ErrNotRSAPrivateKey = errors.New("key is not a valid RSA private key") + ErrNotRSAPublicKey = errors.New("key is not a valid RSA public key") +) + +// ParseRSAPrivateKeyFromPEM parses a PEM encoded PKCS1 or PKCS8 private key +func ParseRSAPrivateKeyFromPEM(key []byte) (*rsa.PrivateKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + var parsedKey interface{} + if parsedKey, err = x509.ParsePKCS1PrivateKey(block.Bytes); err != nil { + if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { + return nil, err + } + } + + var pkey *rsa.PrivateKey + var ok bool + if pkey, ok = parsedKey.(*rsa.PrivateKey); !ok { + return nil, ErrNotRSAPrivateKey + } + + return pkey, nil +} + +// ParseRSAPrivateKeyFromPEMWithPassword parses a PEM encoded PKCS1 or PKCS8 private key protected with password +// +// Deprecated: This function is deprecated and should not be used anymore. It uses the deprecated x509.DecryptPEMBlock +// function, which was deprecated since RFC 1423 is regarded insecure by design. Unfortunately, there is no alternative +// in the Go standard library for now. See https://github.com/golang/go/issues/8860. +func ParseRSAPrivateKeyFromPEMWithPassword(key []byte, password string) (*rsa.PrivateKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + var parsedKey interface{} + + var blockDecrypted []byte + if blockDecrypted, err = x509.DecryptPEMBlock(block, []byte(password)); err != nil { + return nil, err + } + + if parsedKey, err = x509.ParsePKCS1PrivateKey(blockDecrypted); err != nil { + if parsedKey, err = x509.ParsePKCS8PrivateKey(blockDecrypted); err != nil { + return nil, err + } + } + + var pkey *rsa.PrivateKey + var ok bool + if pkey, ok = parsedKey.(*rsa.PrivateKey); !ok { + return nil, ErrNotRSAPrivateKey + } + + return pkey, nil +} + +// ParseRSAPublicKeyFromPEM parses a certificate or a PEM encoded PKCS1 or PKIX public key +func ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { + if cert, err := x509.ParseCertificate(block.Bytes); err == nil { + parsedKey = cert.PublicKey + } else { + if parsedKey, err = x509.ParsePKCS1PublicKey(block.Bytes); err != nil { + return nil, err + } + } + } + + var pkey *rsa.PublicKey + var ok bool + if pkey, ok = parsedKey.(*rsa.PublicKey); !ok { + return nil, ErrNotRSAPublicKey + } + + return pkey, nil +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/signing_method.go b/vendor/github.com/golang-jwt/jwt/v5/signing_method.go new file mode 100644 index 000000000..0d73631c1 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/signing_method.go @@ -0,0 +1,49 @@ +package jwt + +import ( + "sync" +) + +var signingMethods = map[string]func() SigningMethod{} +var signingMethodLock = new(sync.RWMutex) + +// SigningMethod can be used add new methods for signing or verifying tokens. It +// takes a decoded signature as an input in the Verify function and produces a +// signature in Sign. The signature is then usually base64 encoded as part of a +// JWT. +type SigningMethod interface { + Verify(signingString string, sig []byte, key interface{}) error // Returns nil if signature is valid + Sign(signingString string, key interface{}) ([]byte, error) // Returns signature or error + Alg() string // returns the alg identifier for this method (example: 'HS256') +} + +// RegisterSigningMethod registers the "alg" name and a factory function for signing method. +// This is typically done during init() in the method's implementation +func RegisterSigningMethod(alg string, f func() SigningMethod) { + signingMethodLock.Lock() + defer signingMethodLock.Unlock() + + signingMethods[alg] = f +} + +// GetSigningMethod retrieves a signing method from an "alg" string +func GetSigningMethod(alg string) (method SigningMethod) { + signingMethodLock.RLock() + defer signingMethodLock.RUnlock() + + if methodF, ok := signingMethods[alg]; ok { + method = methodF() + } + return +} + +// GetAlgorithms returns a list of registered "alg" names +func GetAlgorithms() (algs []string) { + signingMethodLock.RLock() + defer signingMethodLock.RUnlock() + + for alg := range signingMethods { + algs = append(algs, alg) + } + return +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf b/vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf new file mode 100644 index 000000000..53745d51d --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf @@ -0,0 +1 @@ +checks = ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1023"] diff --git a/vendor/github.com/golang-jwt/jwt/v5/token.go b/vendor/github.com/golang-jwt/jwt/v5/token.go new file mode 100644 index 000000000..352873a2d --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/token.go @@ -0,0 +1,100 @@ +package jwt + +import ( + "crypto" + "encoding/base64" + "encoding/json" +) + +// Keyfunc will be used by the Parse methods as a callback function to supply +// the key for verification. The function receives the parsed, but unverified +// Token. This allows you to use properties in the Header of the token (such as +// `kid`) to identify which key to use. +// +// The returned interface{} may be a single key or a VerificationKeySet containing +// multiple keys. +type Keyfunc func(*Token) (interface{}, error) + +// VerificationKey represents a public or secret key for verifying a token's signature. +type VerificationKey interface { + crypto.PublicKey | []uint8 +} + +// VerificationKeySet is a set of public or secret keys. It is used by the parser to verify a token. +type VerificationKeySet struct { + Keys []VerificationKey +} + +// Token represents a JWT Token. Different fields will be used depending on +// whether you're creating or parsing/verifying a token. +type Token struct { + Raw string // Raw contains the raw token. Populated when you [Parse] a token + Method SigningMethod // Method is the signing method used or to be used + Header map[string]interface{} // Header is the first segment of the token in decoded form + Claims Claims // Claims is the second segment of the token in decoded form + Signature []byte // Signature is the third segment of the token in decoded form. Populated when you Parse a token + Valid bool // Valid specifies if the token is valid. Populated when you Parse/Verify a token +} + +// New creates a new [Token] with the specified signing method and an empty map +// of claims. Additional options can be specified, but are currently unused. +func New(method SigningMethod, opts ...TokenOption) *Token { + return NewWithClaims(method, MapClaims{}, opts...) +} + +// NewWithClaims creates a new [Token] with the specified signing method and +// claims. Additional options can be specified, but are currently unused. +func NewWithClaims(method SigningMethod, claims Claims, opts ...TokenOption) *Token { + return &Token{ + Header: map[string]interface{}{ + "typ": "JWT", + "alg": method.Alg(), + }, + Claims: claims, + Method: method, + } +} + +// SignedString creates and returns a complete, signed JWT. The token is signed +// using the SigningMethod specified in the token. Please refer to +// https://golang-jwt.github.io/jwt/usage/signing_methods/#signing-methods-and-key-types +// for an overview of the different signing methods and their respective key +// types. +func (t *Token) SignedString(key interface{}) (string, error) { + sstr, err := t.SigningString() + if err != nil { + return "", err + } + + sig, err := t.Method.Sign(sstr, key) + if err != nil { + return "", err + } + + return sstr + "." + t.EncodeSegment(sig), nil +} + +// SigningString generates the signing string. This is the most expensive part +// of the whole deal. Unless you need this for something special, just go +// straight for the SignedString. +func (t *Token) SigningString() (string, error) { + h, err := json.Marshal(t.Header) + if err != nil { + return "", err + } + + c, err := json.Marshal(t.Claims) + if err != nil { + return "", err + } + + return t.EncodeSegment(h) + "." + t.EncodeSegment(c), nil +} + +// EncodeSegment encodes a JWT specific base64url encoding with padding +// stripped. In the future, this function might take into account a +// [TokenOption]. Therefore, this function exists as a method of [Token], rather +// than a global function. +func (*Token) EncodeSegment(seg []byte) string { + return base64.RawURLEncoding.EncodeToString(seg) +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/token_option.go b/vendor/github.com/golang-jwt/jwt/v5/token_option.go new file mode 100644 index 000000000..b4ae3badf --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/token_option.go @@ -0,0 +1,5 @@ +package jwt + +// TokenOption is a reserved type, which provides some forward compatibility, +// if we ever want to introduce token creation-related options. +type TokenOption func(*Token) diff --git a/vendor/github.com/golang-jwt/jwt/v5/types.go b/vendor/github.com/golang-jwt/jwt/v5/types.go new file mode 100644 index 000000000..b2655a9e6 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/types.go @@ -0,0 +1,149 @@ +package jwt + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "time" +) + +// TimePrecision sets the precision of times and dates within this library. This +// has an influence on the precision of times when comparing expiry or other +// related time fields. Furthermore, it is also the precision of times when +// serializing. +// +// For backwards compatibility the default precision is set to seconds, so that +// no fractional timestamps are generated. +var TimePrecision = time.Second + +// MarshalSingleStringAsArray modifies the behavior of the ClaimStrings type, +// especially its MarshalJSON function. +// +// If it is set to true (the default), it will always serialize the type as an +// array of strings, even if it just contains one element, defaulting to the +// behavior of the underlying []string. If it is set to false, it will serialize +// to a single string, if it contains one element. Otherwise, it will serialize +// to an array of strings. +var MarshalSingleStringAsArray = true + +// NumericDate represents a JSON numeric date value, as referenced at +// https://datatracker.ietf.org/doc/html/rfc7519#section-2. +type NumericDate struct { + time.Time +} + +// NewNumericDate constructs a new *NumericDate from a standard library time.Time struct. +// It will truncate the timestamp according to the precision specified in TimePrecision. +func NewNumericDate(t time.Time) *NumericDate { + return &NumericDate{t.Truncate(TimePrecision)} +} + +// newNumericDateFromSeconds creates a new *NumericDate out of a float64 representing a +// UNIX epoch with the float fraction representing non-integer seconds. +func newNumericDateFromSeconds(f float64) *NumericDate { + round, frac := math.Modf(f) + return NewNumericDate(time.Unix(int64(round), int64(frac*1e9))) +} + +// MarshalJSON is an implementation of the json.RawMessage interface and serializes the UNIX epoch +// represented in NumericDate to a byte array, using the precision specified in TimePrecision. +func (date NumericDate) MarshalJSON() (b []byte, err error) { + var prec int + if TimePrecision < time.Second { + prec = int(math.Log10(float64(time.Second) / float64(TimePrecision))) + } + truncatedDate := date.Truncate(TimePrecision) + + // For very large timestamps, UnixNano would overflow an int64, but this + // function requires nanosecond level precision, so we have to use the + // following technique to get round the issue: + // + // 1. Take the normal unix timestamp to form the whole number part of the + // output, + // 2. Take the result of the Nanosecond function, which returns the offset + // within the second of the particular unix time instance, to form the + // decimal part of the output + // 3. Concatenate them to produce the final result + seconds := strconv.FormatInt(truncatedDate.Unix(), 10) + nanosecondsOffset := strconv.FormatFloat(float64(truncatedDate.Nanosecond())/float64(time.Second), 'f', prec, 64) + + output := append([]byte(seconds), []byte(nanosecondsOffset)[1:]...) + + return output, nil +} + +// UnmarshalJSON is an implementation of the json.RawMessage interface and +// deserializes a [NumericDate] from a JSON representation, i.e. a +// [json.Number]. This number represents an UNIX epoch with either integer or +// non-integer seconds. +func (date *NumericDate) UnmarshalJSON(b []byte) (err error) { + var ( + number json.Number + f float64 + ) + + if err = json.Unmarshal(b, &number); err != nil { + return fmt.Errorf("could not parse NumericData: %w", err) + } + + if f, err = number.Float64(); err != nil { + return fmt.Errorf("could not convert json number value to float: %w", err) + } + + n := newNumericDateFromSeconds(f) + *date = *n + + return nil +} + +// ClaimStrings is basically just a slice of strings, but it can be either +// serialized from a string array or just a string. This type is necessary, +// since the "aud" claim can either be a single string or an array. +type ClaimStrings []string + +func (s *ClaimStrings) UnmarshalJSON(data []byte) (err error) { + var value interface{} + + if err = json.Unmarshal(data, &value); err != nil { + return err + } + + var aud []string + + switch v := value.(type) { + case string: + aud = append(aud, v) + case []string: + aud = ClaimStrings(v) + case []interface{}: + for _, vv := range v { + vs, ok := vv.(string) + if !ok { + return ErrInvalidType + } + aud = append(aud, vs) + } + case nil: + return nil + default: + return ErrInvalidType + } + + *s = aud + + return +} + +func (s ClaimStrings) MarshalJSON() (b []byte, err error) { + // This handles a special case in the JWT RFC. If the string array, e.g. + // used by the "aud" field, only contains one element, it MAY be serialized + // as a single string. This may or may not be desired based on the ecosystem + // of other JWT library used, so we make it configurable by the variable + // MarshalSingleStringAsArray. + if len(s) == 1 && !MarshalSingleStringAsArray { + return json.Marshal(s[0]) + } + + return json.Marshal([]string(s)) +} diff --git a/vendor/github.com/golang-jwt/jwt/v5/validator.go b/vendor/github.com/golang-jwt/jwt/v5/validator.go new file mode 100644 index 000000000..008ecd871 --- /dev/null +++ b/vendor/github.com/golang-jwt/jwt/v5/validator.go @@ -0,0 +1,316 @@ +package jwt + +import ( + "crypto/subtle" + "fmt" + "time" +) + +// ClaimsValidator is an interface that can be implemented by custom claims who +// wish to execute any additional claims validation based on +// application-specific logic. The Validate function is then executed in +// addition to the regular claims validation and any error returned is appended +// to the final validation result. +// +// type MyCustomClaims struct { +// Foo string `json:"foo"` +// jwt.RegisteredClaims +// } +// +// func (m MyCustomClaims) Validate() error { +// if m.Foo != "bar" { +// return errors.New("must be foobar") +// } +// return nil +// } +type ClaimsValidator interface { + Claims + Validate() error +} + +// Validator is the core of the new Validation API. It is automatically used by +// a [Parser] during parsing and can be modified with various parser options. +// +// The [NewValidator] function should be used to create an instance of this +// struct. +type Validator struct { + // leeway is an optional leeway that can be provided to account for clock skew. + leeway time.Duration + + // timeFunc is used to supply the current time that is needed for + // validation. If unspecified, this defaults to time.Now. + timeFunc func() time.Time + + // requireExp specifies whether the exp claim is required + requireExp bool + + // verifyIat specifies whether the iat (Issued At) claim will be verified. + // According to https://www.rfc-editor.org/rfc/rfc7519#section-4.1.6 this + // only specifies the age of the token, but no validation check is + // necessary. However, if wanted, it can be checked if the iat is + // unrealistic, i.e., in the future. + verifyIat bool + + // expectedAud contains the audience this token expects. Supplying an empty + // string will disable aud checking. + expectedAud string + + // expectedIss contains the issuer this token expects. Supplying an empty + // string will disable iss checking. + expectedIss string + + // expectedSub contains the subject this token expects. Supplying an empty + // string will disable sub checking. + expectedSub string +} + +// NewValidator can be used to create a stand-alone validator with the supplied +// options. This validator can then be used to validate already parsed claims. +// +// Note: Under normal circumstances, explicitly creating a validator is not +// needed and can potentially be dangerous; instead functions of the [Parser] +// class should be used. +// +// The [Validator] is only checking the *validity* of the claims, such as its +// expiration time, but it does NOT perform *signature verification* of the +// token. +func NewValidator(opts ...ParserOption) *Validator { + p := NewParser(opts...) + return p.validator +} + +// Validate validates the given claims. It will also perform any custom +// validation if claims implements the [ClaimsValidator] interface. +// +// Note: It will NOT perform any *signature verification* on the token that +// contains the claims and expects that the [Claim] was already successfully +// verified. +func (v *Validator) Validate(claims Claims) error { + var ( + now time.Time + errs []error = make([]error, 0, 6) + err error + ) + + // Check, if we have a time func + if v.timeFunc != nil { + now = v.timeFunc() + } else { + now = time.Now() + } + + // We always need to check the expiration time, but usage of the claim + // itself is OPTIONAL by default. requireExp overrides this behavior + // and makes the exp claim mandatory. + if err = v.verifyExpiresAt(claims, now, v.requireExp); err != nil { + errs = append(errs, err) + } + + // We always need to check not-before, but usage of the claim itself is + // OPTIONAL. + if err = v.verifyNotBefore(claims, now, false); err != nil { + errs = append(errs, err) + } + + // Check issued-at if the option is enabled + if v.verifyIat { + if err = v.verifyIssuedAt(claims, now, false); err != nil { + errs = append(errs, err) + } + } + + // If we have an expected audience, we also require the audience claim + if v.expectedAud != "" { + if err = v.verifyAudience(claims, v.expectedAud, true); err != nil { + errs = append(errs, err) + } + } + + // If we have an expected issuer, we also require the issuer claim + if v.expectedIss != "" { + if err = v.verifyIssuer(claims, v.expectedIss, true); err != nil { + errs = append(errs, err) + } + } + + // If we have an expected subject, we also require the subject claim + if v.expectedSub != "" { + if err = v.verifySubject(claims, v.expectedSub, true); err != nil { + errs = append(errs, err) + } + } + + // Finally, we want to give the claim itself some possibility to do some + // additional custom validation based on a custom Validate function. + cvt, ok := claims.(ClaimsValidator) + if ok { + if err := cvt.Validate(); err != nil { + errs = append(errs, err) + } + } + + if len(errs) == 0 { + return nil + } + + return joinErrors(errs...) +} + +// verifyExpiresAt compares the exp claim in claims against cmp. This function +// will succeed if cmp < exp. Additional leeway is taken into account. +// +// If exp is not set, it will succeed if the claim is not required, +// otherwise ErrTokenRequiredClaimMissing will be returned. +// +// Additionally, if any error occurs while retrieving the claim, e.g., when its +// the wrong type, an ErrTokenUnverifiable error will be returned. +func (v *Validator) verifyExpiresAt(claims Claims, cmp time.Time, required bool) error { + exp, err := claims.GetExpirationTime() + if err != nil { + return err + } + + if exp == nil { + return errorIfRequired(required, "exp") + } + + return errorIfFalse(cmp.Before((exp.Time).Add(+v.leeway)), ErrTokenExpired) +} + +// verifyIssuedAt compares the iat claim in claims against cmp. This function +// will succeed if cmp >= iat. Additional leeway is taken into account. +// +// If iat is not set, it will succeed if the claim is not required, +// otherwise ErrTokenRequiredClaimMissing will be returned. +// +// Additionally, if any error occurs while retrieving the claim, e.g., when its +// the wrong type, an ErrTokenUnverifiable error will be returned. +func (v *Validator) verifyIssuedAt(claims Claims, cmp time.Time, required bool) error { + iat, err := claims.GetIssuedAt() + if err != nil { + return err + } + + if iat == nil { + return errorIfRequired(required, "iat") + } + + return errorIfFalse(!cmp.Before(iat.Add(-v.leeway)), ErrTokenUsedBeforeIssued) +} + +// verifyNotBefore compares the nbf claim in claims against cmp. This function +// will return true if cmp >= nbf. Additional leeway is taken into account. +// +// If nbf is not set, it will succeed if the claim is not required, +// otherwise ErrTokenRequiredClaimMissing will be returned. +// +// Additionally, if any error occurs while retrieving the claim, e.g., when its +// the wrong type, an ErrTokenUnverifiable error will be returned. +func (v *Validator) verifyNotBefore(claims Claims, cmp time.Time, required bool) error { + nbf, err := claims.GetNotBefore() + if err != nil { + return err + } + + if nbf == nil { + return errorIfRequired(required, "nbf") + } + + return errorIfFalse(!cmp.Before(nbf.Add(-v.leeway)), ErrTokenNotValidYet) +} + +// verifyAudience compares the aud claim against cmp. +// +// If aud is not set or an empty list, it will succeed if the claim is not required, +// otherwise ErrTokenRequiredClaimMissing will be returned. +// +// Additionally, if any error occurs while retrieving the claim, e.g., when its +// the wrong type, an ErrTokenUnverifiable error will be returned. +func (v *Validator) verifyAudience(claims Claims, cmp string, required bool) error { + aud, err := claims.GetAudience() + if err != nil { + return err + } + + if len(aud) == 0 { + return errorIfRequired(required, "aud") + } + + // use a var here to keep constant time compare when looping over a number of claims + result := false + + var stringClaims string + for _, a := range aud { + if subtle.ConstantTimeCompare([]byte(a), []byte(cmp)) != 0 { + result = true + } + stringClaims = stringClaims + a + } + + // case where "" is sent in one or many aud claims + if stringClaims == "" { + return errorIfRequired(required, "aud") + } + + return errorIfFalse(result, ErrTokenInvalidAudience) +} + +// verifyIssuer compares the iss claim in claims against cmp. +// +// If iss is not set, it will succeed if the claim is not required, +// otherwise ErrTokenRequiredClaimMissing will be returned. +// +// Additionally, if any error occurs while retrieving the claim, e.g., when its +// the wrong type, an ErrTokenUnverifiable error will be returned. +func (v *Validator) verifyIssuer(claims Claims, cmp string, required bool) error { + iss, err := claims.GetIssuer() + if err != nil { + return err + } + + if iss == "" { + return errorIfRequired(required, "iss") + } + + return errorIfFalse(iss == cmp, ErrTokenInvalidIssuer) +} + +// verifySubject compares the sub claim against cmp. +// +// If sub is not set, it will succeed if the claim is not required, +// otherwise ErrTokenRequiredClaimMissing will be returned. +// +// Additionally, if any error occurs while retrieving the claim, e.g., when its +// the wrong type, an ErrTokenUnverifiable error will be returned. +func (v *Validator) verifySubject(claims Claims, cmp string, required bool) error { + sub, err := claims.GetSubject() + if err != nil { + return err + } + + if sub == "" { + return errorIfRequired(required, "sub") + } + + return errorIfFalse(sub == cmp, ErrTokenInvalidSubject) +} + +// errorIfFalse returns the error specified in err, if the value is true. +// Otherwise, nil is returned. +func errorIfFalse(value bool, err error) error { + if value { + return nil + } else { + return err + } +} + +// errorIfRequired returns an ErrTokenRequiredClaimMissing error if required is +// true. Otherwise, nil is returned. +func errorIfRequired(required bool, claim string) error { + if required { + return newError(fmt.Sprintf("%s claim is required", claim), ErrTokenRequiredClaimMissing) + } else { + return nil + } +} From e55978b927bb403ee6e2dc4cd093aacd592933d1 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 2 Apr 2024 21:56:28 +0530 Subject: [PATCH 128/235] Resolve few TODOs, fix errors from `make check` --- internal/controller/etcd/reconciler.go | 1 - internal/controller/utils/reconciler.go | 13 -------- internal/operator/rolebinding/rolebinding.go | 13 -------- internal/operator/statefulset/builder.go | 3 +- internal/operator/statefulset/statefulset.go | 31 ++------------------ internal/webhook/sentinel/handler_test.go | 8 ++--- 6 files changed, 8 insertions(+), 61 deletions(-) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 807b8ac79..c947edf0d 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -70,7 +70,6 @@ func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, iv imagev type reconcileFn func(ctx component.OperatorContext, objectKey client.ObjectKey) ctrlutils.ReconcileStepResult -// TODO: where/how is this being used? // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds/status,verbs=get;create;update;patch // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update;patch;delete;deletecollection diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index e568223b9..bc5ca0f10 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -40,19 +40,6 @@ func CreateImageVector() (imagevector.ImageVector, error) { return imageVector, nil } -// ContainsFinalizer checks if an object has a finalizer present on it. -// TODO: With the controller-runtime version 0.16.x onwards this is provided by controllerutil.ContainsFinalizer. -// TODO: Remove this function once we move to this version. -func ContainsFinalizer(o client.Object, finalizer string) bool { - finalizers := o.GetFinalizers() - for _, f := range finalizers { - if f == finalizer { - return true - } - } - return false -} - // GetLatestEtcd returns the latest version of the Etcd object. func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ReconcileStepResult { if err := client.Get(ctx, objectKey, etcd); err != nil { diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index d034d4ec7..fd0db1447 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -79,19 +79,6 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return nil } -// Exists checks if the role binding exists for the given Etcd. -// TODO: This method is not used. Remove it if not needed. -func (r _resource) Exists(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (bool, error) { - rb := &rbacv1.RoleBinding{} - if err := r.client.Get(ctx, getObjectKey(etcd), rb); err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - // TriggerDelete triggers the deletion of the role binding for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index afc9434d2..3e2b29718 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -649,8 +649,7 @@ func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { RunAsGroup: pointer.Int64(65532), RunAsNonRoot: pointer.Bool(true), RunAsUser: pointer.Int64(65532), - // TODO: is this necessary? - FSGroup: pointer.Int64(65532), + FSGroup: pointer.Int64(65532), } } diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 2af771f2f..b882141bf 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -10,16 +10,15 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" - corev1 "k8s.io/api/core/v1" - "k8s.io/component-base/featuregate" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -175,7 +174,7 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru } return fmt.Errorf("peer URL TLS not enabled for #%d members for etcd: %v, requeuing reconcile request", *existingSts.Spec.Replicas, etcd.GetNamespaceName()) } - ctx.Logger.Info("Peer URL TLS has been enabled for all members") + ctx.Logger.Info("Peer URL TLS has been enabled for all currently running members") return nil } @@ -198,30 +197,6 @@ func isPeerTLSEnablementPending(peerTLSEnabledStatusFromMembers bool, etcd *drui return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } -// TODO: this function is not used. Remove it if not needed. -func deleteAllStsPods(ctx component.OperatorContext, cl client.Client, opName string, sts *appsv1.StatefulSet) error { - // Get all Pods belonging to the StatefulSet - podList := &corev1.PodList{} - listOpts := []client.ListOption{ - client.InNamespace(sts.Namespace), - client.MatchingLabels(sts.Spec.Selector.MatchLabels), - } - - if err := cl.List(ctx, podList, listOpts...); err != nil { - ctx.Logger.Error(err, "Failed to list pods for StatefulSet", "StatefulSet", client.ObjectKeyFromObject(sts), "operation", opName) - return err - } - - for _, pod := range podList.Items { - if err := cl.Delete(ctx, &pod); err != nil { - ctx.Logger.Error(err, "Failed to delete pod", "Pod", pod.Name, "Namespace", pod.Namespace, "operation", opName) - return err - } - } - - return nil -} - func emptyStatefulSet(etcd *druidv1alpha1.Etcd) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index 88b3e18de..9bfa4ccb7 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -40,8 +40,8 @@ const ( ) var ( - internalErr = errors.New("test internal error") - apiInternalErr = apierrors.NewInternalError(internalErr) + errInternal = errors.New("test internal error") + apiInternalErr = apierrors.NewInternalError(errInternal) apiNotFoundErr = apierrors.NewNotFound(schema.GroupResource{}, "") reconcilerServiceAccount = "etcd-druid-sa" exemptServiceAccounts = []string{"exempt-sa-1"} @@ -148,7 +148,7 @@ func TestUnexpectedResourceType(t *testing.T) { }) g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(string(resp.Result.Reason)).To(Equal(fmt.Sprintf("unexpected resource type: coordination.k8s.io/Unknown"))) + g.Expect(string(resp.Result.Reason)).To(Equal("unexpected resource type: coordination.k8s.io/Unknown")) } func TestMissingResourcePartOfLabel(t *testing.T) { @@ -403,7 +403,7 @@ func TestEtcdGetFailures(t *testing.T) { name: "error in getting etcd", etcdGetErr: apiInternalErr, expectedAllowed: false, - expectedMessage: internalErr.Error(), + expectedMessage: errInternal.Error(), expectedCode: http.StatusInternalServerError, }, } From 049a57a205bafeab0a89609cab06d22b312f67cd Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 3 Apr 2024 09:37:51 +0530 Subject: [PATCH 129/235] upgraded golang version to 1.22.1 --- .ci/pipeline_definitions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/pipeline_definitions b/.ci/pipeline_definitions index cebb90752..25c458a74 100644 --- a/.ci/pipeline_definitions +++ b/.ci/pipeline_definitions @@ -35,13 +35,13 @@ etcd-druid: teamname: 'gardener/etcd-druid-maintainers' steps: check: - image: 'golang:1.22.0' + image: 'golang:1.22.1' test: - image: 'golang:1.22.0' + image: 'golang:1.22.1' test_integration: - image: 'golang:1.22.0' + image: 'golang:1.22.1' build: - image: 'golang:1.22.0' + image: 'golang:1.22.1' output_dir: 'binary' jobs: From 81c0dcee7710b68848ae4cc12f1658be78d3b556 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 3 Apr 2024 10:18:16 +0530 Subject: [PATCH 130/235] Streamline the running of tests --- Makefile | 15 ++++++--------- hack/test-go.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++ hack/test.sh | 38 ++++++++++++++++++++------------------ 3 files changed, 70 insertions(+), 27 deletions(-) create mode 100755 hack/test-go.sh diff --git a/Makefile b/Makefile index d331c634d..56a02ce20 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,8 @@ docker-push: # Run tests .PHONY: test test: $(GINKGO) - # run ginkgo unit tests. These will be ported to golang native tests over a period of time. +test: $(GINKGO) + @# run ginkgo unit tests. These will be ported to golang native tests over a period of time. @"$(REPO_ROOT)/hack/test.sh" ./api/... \ ./internal/controller/etcdcopybackupstask/... \ ./internal/controller/predicate/... \ @@ -115,9 +116,8 @@ test: $(GINKGO) ./internal/controller/utils/... \ ./internal/mapper/... \ ./internal/metrics/... - # run the golang native unit tests. - @go test -v -coverprofile cover.out ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... - @go tool cover -func=cover.out + @# run the golang native unit tests. + @TEST_COV="true" "$(REPO_ROOT)/hack/test-go.sh" ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... .PHONY: test-cov test-cov: $(GINKGO) $(SETUP_ENVTEST) @@ -133,11 +133,8 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) .PHONY: test-integration test-integration: set-permissions $(GINKGO) $(SETUP_ENVTEST) - @"$(REPO_ROOT)/hack/test.sh" ./test/integration/... - @export KUBEBUILDER_ASSETS="$(${SETUP_ENVTEST} --arch=amd64 use --use-env -p path 1.22)" - @echo "using envtest tools installed at '${KUBEBUILDER_ASSETS}'" - @export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=2m - @go test -v ./test/it/... + @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test.sh" ./test/integration/... + @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test-go.sh" ./test/it/... .PHONY: update-dependencies update-dependencies: diff --git a/hack/test-go.sh b/hack/test-go.sh new file mode 100755 index 000000000..76f9a54e3 --- /dev/null +++ b/hack/test-go.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.22"} + +if ${SETUP_ENVTEST:-false}; then + echo "> Installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest" + if ! command -v setup-envtest &> /dev/null ; then + >&2 echo "setup-envtest not available" + exit 1 + fi + + ARCH= + # if using M1 macbook, use amd64 architecture build, as suggested in + # https://github.com/kubernetes-sigs/controller-runtime/issues/1657#issuecomment-988484517 + if [[ $(uname) == 'Darwin' && $(uname -m) == 'arm64' ]]; then + ARCH='--arch=amd64' + fi + + # --use-env allows overwriting the envtest tools path via the KUBEBUILDER_ASSETS env var just like it was before + export KUBEBUILDER_ASSETS="$(setup-envtest ${ARCH} use --use-env -p path ${ENVTEST_K8S_VERSION})" + echo "using envtest tools installed at '$KUBEBUILDER_ASSETS'" + export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=2m +fi + +echo "> Go tests" + +if ${TEST_COV:-false}; then + output_dir=test/output + coverprofile_file=coverprofile.out + mkdir -p test/output + go test -v -coverprofile=cover.out $@ + go tool cover -func=cover.out + exit 0 +fi + +go test -v $@ diff --git a/hack/test.sh b/hack/test.sh index bc08155e2..c5ca05fe9 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -9,36 +9,38 @@ set -o pipefail ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.22"} -echo "> Installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest if necessary" -if ! command -v setup-envtest &> /dev/null ; then - >&2 echo "setup-envtest not available" - exit 1 +if ${SETUP_ENVTEST:-false}; then + echo "> Installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest" + if ! command -v setup-envtest &> /dev/null ; then + >&2 echo "setup-envtest not available" + exit 1 + fi + + ARCH= + # if using M1 macbook, use amd64 architecture build, as suggested in + # https://github.com/kubernetes-sigs/controller-runtime/issues/1657#issuecomment-988484517 + if [[ $(uname) == 'Darwin' && $(uname -m) == 'arm64' ]]; then + ARCH='--arch=amd64' + fi + + # --use-env allows overwriting the envtest tools path via the KUBEBUILDER_ASSETS env var just like it was before + export KUBEBUILDER_ASSETS="$(setup-envtest ${ARCH} use --use-env -p path ${ENVTEST_K8S_VERSION})" + echo "using envtest tools installed at '$KUBEBUILDER_ASSETS'" + export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=2m fi -ARCH= -# if using M1 macbook, use amd64 architecture build, as suggested in -# https://github.com/kubernetes-sigs/controller-runtime/issues/1657#issuecomment-988484517 -if [[ $(uname) == 'Darwin' && $(uname -m) == 'arm64' ]]; then - ARCH='--arch=amd64' -fi - -# --use-env allows overwriting the envtest tools path via the KUBEBUILDER_ASSETS env var just like it was before -export KUBEBUILDER_ASSETS="$(setup-envtest ${ARCH} use --use-env -p path ${ENVTEST_K8S_VERSION})" -echo "using envtest tools installed at '$KUBEBUILDER_ASSETS'" - -echo "> Tests" - if [[ $(uname) == 'Darwin' ]]; then SED_BIN="gsed" else SED_BIN="sed" fi -export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=2m export GOMEGA_DEFAULT_EVENTUALLY_TIMEOUT=5s export GOMEGA_DEFAULT_EVENTUALLY_POLLING_INTERVAL=200ms GINKGO_COMMON_FLAGS="-r -timeout=1h0m0s --randomize-all --randomize-suites --fail-on-pending --show-node-events" +echo "> Ginkgo tests" + if ${TEST_COV:-false}; then output_dir=test/output coverprofile_file=coverprofile.out From c11d7e6bbd77885e637e0d3a851b52c449280aa7 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 5 Apr 2024 18:31:54 +0530 Subject: [PATCH 131/235] Fix e2e tests --- .../templates/controller-deployment.yaml | 3 ++ charts/druid/values.yaml | 1 + hack/e2e-test/run-e2e-test.sh | 1 + internal/controller/compaction/reconciler.go | 10 ++-- internal/operator/statefulset/builder.go | 4 +- internal/operator/statefulset/constants.go | 4 +- internal/operator/statefulset/stsmatcher.go | 20 +++---- skaffold.yaml | 7 +++ test/e2e/etcd_backup_test.go | 44 +++++++++------ test/e2e/etcd_compaction_test.go | 35 +++++++----- test/e2e/etcd_multi_node_test.go | 54 ++++++++++--------- test/e2e/suite_test.go | 6 +++ .../controllers/compaction/reconciler_test.go | 15 ++---- test/utils/statefulset.go | 16 +++--- 14 files changed, 132 insertions(+), 88 deletions(-) diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index b42ccb3cf..0e66fa082 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -26,6 +26,9 @@ spec: - --enable-leader-election=true - --ignore-operation-annotation={{ .Values.ignoreOperationAnnotation }} - --etcd-workers=3 + {{- if .Values.etcdStatusSyncPeriod }} + - --etcd-status-sync-period={{ .Values.etcdStatusSyncPeriod }} + {{- end }} {{- if .Values.enableBackupCompaction }} - --enable-backup-compaction={{ .Values.enableBackupCompaction }} {{- end }} diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index 3fb828c2f..c82a3f12c 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -6,6 +6,7 @@ image: imagePullPolicy: IfNotPresent replicas: 1 ignoreOperationAnnotation: false +# etcdStatusSyncPeriod: 15s # enableBackupCompaction: true # eventsThreshold: 15 # metricsScrapeWaitDuration: "10s" diff --git a/hack/e2e-test/run-e2e-test.sh b/hack/e2e-test/run-e2e-test.sh index bfe995c21..f35b086c7 100755 --- a/hack/e2e-test/run-e2e-test.sh +++ b/hack/e2e-test/run-e2e-test.sh @@ -93,6 +93,7 @@ function deploy { echo "-------------------" echo "Deploying Druid" echo "-------------------" + export DRUID_E2E_TEST=true skaffold_run_or_deploy -m etcd-druid -n $TEST_ID deployed="true" fi diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index ed4f20a8a..fe54e1a5c 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -15,6 +15,7 @@ import ( ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" druidmetrics "github.com/gardener/etcd-druid/internal/metrics" + "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -357,7 +358,7 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu vms := []v1.VolumeMount{ { Name: "etcd-workspace-dir", - MountPath: "/var/etcd/data", + MountPath: statefulset.EtcdDataVolumeMountPath, }, } @@ -381,12 +382,12 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu case utils.GCS: vms = append(vms, v1.VolumeMount{ Name: "etcd-backup", - MountPath: "/var/.gcp/", + MountPath: statefulset.GCSBackupVolumeMountPath, }) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: vms = append(vms, v1.VolumeMount{ Name: "etcd-backup", - MountPath: "/var/etcd-backup/", + MountPath: statefulset.NonGCSProviderBackupVolumeMountPath, }) } @@ -434,8 +435,7 @@ func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr. Name: "etcd-backup", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: storeValues.SecretRef.Name, - DefaultMode: pointer.Int32(0640), + SecretName: storeValues.SecretRef.Name, }, }, }) diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 3e2b29718..f2d5840c3 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -325,12 +325,12 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { case utils.GCS: return &corev1.VolumeMount{ Name: providerBackupVolumeName, - MountPath: gcsBackupVolumeMountPath, + MountPath: GCSBackupVolumeMountPath, } case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: return &corev1.VolumeMount{ Name: providerBackupVolumeName, - MountPath: nonGCSProviderBackupVolumeMountPath, + MountPath: NonGCSProviderBackupVolumeMountPath, } } return nil diff --git a/internal/operator/statefulset/constants.go b/internal/operator/statefulset/constants.go index 90dc26b91..21f743c63 100644 --- a/internal/operator/statefulset/constants.go +++ b/internal/operator/statefulset/constants.go @@ -32,8 +32,8 @@ const ( backupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" backupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" backupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" - gcsBackupVolumeMountPath = "/var/.gcp/" - nonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" + GCSBackupVolumeMountPath = "/var/.gcp/" + NonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" ) const ( diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 1c9db1b0c..7a853cf42 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -12,7 +12,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/utils" - utils2 "github.com/gardener/etcd-druid/test/utils" + testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -86,15 +86,15 @@ func (s StatefulSetMatcher) matchSTSObjectMeta() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "Name": Equal(s.etcd.Name), "Namespace": Equal(s.etcd.Namespace), - "OwnerReferences": utils2.MatchEtcdOwnerReference(s.etcd.Name, s.etcd.UID), - "Labels": utils2.MatchResourceLabels(getStatefulSetLabels(s.etcd.Name)), + "OwnerReferences": testutils.MatchEtcdOwnerReference(s.etcd.Name, s.etcd.UID), + "Labels": testutils.MatchResourceLabels(getStatefulSetLabels(s.etcd.Name)), }) } func (s StatefulSetMatcher) matchSpec() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "Replicas": PointTo(Equal(s.replicas)), - "Selector": utils2.MatchSpecLabelSelector(s.etcd.GetDefaultLabels()), + "Selector": testutils.MatchSpecLabelSelector(s.etcd.GetDefaultLabels()), "PodManagementPolicy": Equal(appsv1.ParallelPodManagement), "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), @@ -132,9 +132,9 @@ func (s StatefulSetMatcher) matchPodTemplateSpec() gomegatypes.GomegaMatcher { func (s StatefulSetMatcher) matchPodObjectMeta() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ - "Labels": utils2.MatchResourceLabels(utils.MergeMaps[string, string](getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), - "Annotations": utils2.MatchResourceAnnotations(utils.MergeMaps[string, string](s.etcd.Spec.Annotations, map[string]string{ - "checksum/etcd-configmap": utils2.TestConfigMapCheckSum, + "Labels": testutils.MatchResourceLabels(utils.MergeMaps[string, string](getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), + "Annotations": testutils.MatchResourceAnnotations(utils.MergeMaps[string, string](s.etcd.Spec.Annotations, map[string]string{ + "checksum/etcd-configmap": testutils.TestConfigMapCheckSum, })), }) } @@ -239,7 +239,7 @@ func (s StatefulSetMatcher) matchEtcdContainerVolMounts() gomegatypes.GomegaMatc } func (s StatefulSetMatcher) matchBackupRestoreContainer() gomegatypes.GomegaMatcher { - containerResources := utils2.TypeDeref(s.etcd.Spec.Backup.Resources, defaultTestContainerResources) + containerResources := testutils.TypeDeref(s.etcd.Spec.Backup.Resources, defaultTestContainerResources) return MatchFields(IgnoreExtras, Fields{ "Name": Equal("backup-restore"), "Image": Equal(s.etcdBRImage), @@ -377,9 +377,9 @@ func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.Gomega } } case utils.GCS: - return matchVolMount(providerBackupVolumeName, gcsBackupVolumeMountPath) + return matchVolMount(providerBackupVolumeName, GCSBackupVolumeMountPath) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - return matchVolMount(providerBackupVolumeName, nonGCSProviderBackupVolumeMountPath) + return matchVolMount(providerBackupVolumeName, NonGCSProviderBackupVolumeMountPath) } return nil } diff --git a/skaffold.yaml b/skaffold.yaml index 42c33ba01..5dfa732a7 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -27,6 +27,13 @@ deploy: eventsThreshold: 15 metricsScrapeWaitDuration: "30s" profiles: +- name: e2e-test + activation: + - env: "DRUID_E2E_TEST=true" + patches: + - op: add + path: /deploy/helm/releases/0/setValues/etcdStatusSyncPeriod + value: "5s" - name: do-not-use-feature-gates activation: - env: "USE_ETCD_DRUID_FEATURE_GATES=false" diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index 337b4e1aa..17f071c09 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -11,6 +11,7 @@ import ( brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/test/matchers" "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" @@ -51,17 +52,31 @@ var _ = Describe("Etcd Backup", func() { Expect(err).ShouldNot(HaveOccurred()) etcdName = fmt.Sprintf("etcd-%s", provider.Name) - storageContainer = getEnvAndExpectNoError(envStorageContainer) + }) - snapstoreProvider := provider.Storage.Provider - store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) - Expect(err).ShouldNot(HaveOccurred()) + AfterEach(func() { + ctx, cancelFunc := context.WithTimeout(parentCtx, 10*time.Minute) + defer cancelFunc() - // purge any existing backups in bucket - Expect(purgeSnapstore(store)).To(Succeed()) + By("Delete debug pod") + etcd := getDefaultEtcd(etcdName, namespace, storageContainer, storePrefix, provider) + debugPod := getDebugPod(etcd) + Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) - Expect(deployBackupSecret(parentCtx, cl, logger, provider, etcdNamespace, storageContainer)) + By("Purge etcd") + purgeEtcd(ctx, cl, providers) + + By("Purge snapstore") + snapstoreProvider := provider.Storage.Provider + if snapstoreProvider == utils.Local { + purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) + defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) + } else { + store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) + Expect(err).ShouldNot(HaveOccurred()) + Expect(purgeSnapstore(store)).To(Succeed()) + } }) It("Should create, test backup and delete etcd with backup", func() { @@ -174,12 +189,6 @@ var _ = Describe("Etcd Backup", func() { for i := 1; i <= 15; i++ { Expect(keyValueMap[fmt.Sprintf("%s-%d", etcdKeyPrefix, i)]).To(Equal(fmt.Sprintf("%s-%d", etcdValuePrefix, i))) } - - By("Delete debug pod") - Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) - - By("Delete etcd") - deleteAndCheckEtcd(ctx, cl, objLogger, etcd, singleNodeEtcdTimeout) }) }) } @@ -310,14 +319,19 @@ func deleteAndCheckEtcd(ctx context.Context, cl client.Client, logger logr.Logge } func purgeEtcdPVCs(ctx context.Context, cl client.Client, etcdName string) { - r, _ := k8s_labels.NewRequirement("instance", selection.Equals, []string{etcdName}) + r1, err := k8s_labels.NewRequirement(v1alpha1.LabelPartOfKey, selection.Equals, []string{etcdName}) + ExpectWithOffset(1, err).ShouldNot(HaveOccurred()) + r2, err := k8s_labels.NewRequirement(v1alpha1.LabelManagedByKey, selection.Equals, []string{v1alpha1.LabelManagedByValue}) + ExpectWithOffset(1, err).ShouldNot(HaveOccurred()) + pvc := &corev1.PersistentVolumeClaim{} delOptions := client.DeleteOptions{} delOptions.ApplyOptions([]client.DeleteOption{client.PropagationPolicy(metav1.DeletePropagationForeground)}) + logger.Info("Deleting PVCs") ExpectWithOffset(1, client.IgnoreNotFound(cl.DeleteAllOf(ctx, pvc, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ Namespace: namespace, - LabelSelector: k8s_labels.NewSelector().Add(*r), + LabelSelector: k8s_labels.NewSelector().Add(*r1, *r2), }, DeleteOptions: delOptions, }))).ShouldNot(HaveOccurred()) diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index 6d2398745..58e0cd0a9 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -10,6 +10,7 @@ import ( "time" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" + "github.com/gardener/etcd-druid/internal/utils" batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/types" @@ -45,17 +46,31 @@ var _ = Describe("Etcd Compaction", func() { Expect(err).ShouldNot(HaveOccurred()) etcdName = fmt.Sprintf("etcd-%s", provider.Name) - storageContainer = getEnvAndExpectNoError(envStorageContainer) + }) - snapstoreProvider := provider.Storage.Provider - store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) - Expect(err).ShouldNot(HaveOccurred()) + AfterEach(func() { + ctx, cancelFunc := context.WithTimeout(parentCtx, 10*time.Minute) + defer cancelFunc() + + By("Delete debug pod") + etcd := getDefaultEtcd(etcdName, namespace, storageContainer, storePrefix, provider) + debugPod := getDebugPod(etcd) + Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) - // purge any existing backups in bucket - Expect(purgeSnapstore(store)).To(Succeed()) + By("Purge etcd") + purgeEtcd(ctx, cl, providers) - Expect(deployBackupSecret(parentCtx, cl, logger, provider, etcdNamespace, storageContainer)) + By("Purge snapstore") + snapstoreProvider := provider.Storage.Provider + if snapstoreProvider == utils.Local { + purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) + defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) + } else { + store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) + Expect(err).ShouldNot(HaveOccurred()) + Expect(purgeSnapstore(store)).To(Succeed()) + } }) It("should test compaction on backup", func() { @@ -167,12 +182,6 @@ var _ = Describe("Etcd Compaction", func() { Expect(err).ShouldNot(HaveOccurred()) Expect(len(latestSnapshotsAfterPopulate.DeltaSnapshots)).Should(BeNumerically(">", 0)) - - By("Delete debug pod") - Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) - - By("Delete etcd") - deleteAndCheckEtcd(ctx, cl, objLogger, etcd, singleNodeEtcdTimeout) }) }) } diff --git a/test/e2e/etcd_multi_node_test.go b/test/e2e/etcd_multi_node_test.go index 09a2455d6..729500c84 100644 --- a/test/e2e/etcd_multi_node_test.go +++ b/test/e2e/etcd_multi_node_test.go @@ -49,28 +49,23 @@ var _ = Describe("Etcd", func() { provider = providers[0] etcdName = fmt.Sprintf("etcd-%s", provider.Name) - storageContainer = getEnvAndExpectNoError(envStorageContainer) + }) - snapstoreProvider := provider.Storage.Provider + AfterEach(func() { + By("Purge etcd") + purgeEtcd(parentCtx, cl, providers) + By("Purge snapstore") + snapstoreProvider := provider.Storage.Provider if snapstoreProvider == utils.Local { purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) } else { store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) Expect(err).ShouldNot(HaveOccurred()) - // purge any existing backups in bucket Expect(purgeSnapstore(store)).To(Succeed()) } - - Expect(deployBackupSecret(parentCtx, cl, logger, provider, etcdNamespace, storageContainer)) - - }) - - AfterEach(func() { - // remove etcd objects if any old etcd objects exists. - purgeEtcd(parentCtx, cl, providers) }) Context("when multi-node is configured", func() { @@ -93,11 +88,12 @@ var _ = Describe("Etcd", func() { etcd.Spec.Replicas = multiNodeEtcdReplicas updateAndCheckEtcd(ctx, cl, logger, etcd, multiNodeEtcdTimeout) - By("Zero downtime rolling updates") + By("Deploy etcd zero downtime validator job") job := startEtcdZeroDownTimeValidatorJob(ctx, cl, etcd, "rolling-update") // this defer ensures to remove the job. defer cleanUpTestHelperJob(ctx, cl, job.Name) + By("Zero downtime rolling updates") Expect(cl.Get(ctx, client.ObjectKeyFromObject(etcd), etcd)).To(Succeed()) // trigger rolling update by updating etcd quota etcd.Spec.Etcd.Quota.Add(*resource.NewMilliQuantity(int64(10), resource.DecimalSI)) @@ -121,6 +117,7 @@ var _ = Describe("Etcd", func() { By("Member restart with data-dir/pvc intact") objLogger.Info("Delete one member pod") deletePod(ctx, cl, objLogger, etcd, fmt.Sprintf("%s-2", etcdName)) + checkForUnreadyEtcdMembers(ctx, cl, objLogger, etcd, 30*time.Second) checkEtcdReady(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) By("Single member restoration") @@ -128,6 +125,7 @@ var _ = Describe("Etcd", func() { debugPod := createDebugPod(ctx, etcd) objLogger.Info("Delete member dir of one member pod") deleteMemberDir(ctx, cl, objLogger, etcd, debugPod.Name, debugPod.Spec.Containers[0].Name) + checkForUnreadyEtcdMembers(ctx, cl, objLogger, etcd, 30*time.Second) checkEtcdReady(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) By("Delete debug pod") @@ -153,9 +151,6 @@ var _ = Describe("Etcd", func() { Expect(cl.Get(ctx, client.ObjectKeyFromObject(etcd), etcd)).To(Succeed()) etcd.Spec.Replicas = 3 updateAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) - - By("Deleting etcd") - deleteAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) }) It("should scale a single-node etcd (TLS not enabled for peerUrl) to a multi-node etcd cluster (TLS enabled for peerUrl)", func() { @@ -173,9 +168,6 @@ var _ = Describe("Etcd", func() { etcd.Spec.Replicas = 3 etcd.Spec.Etcd.PeerUrlTLS = getPeerTls(provider.Suffix) updateAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) - - By("Deleting the single-node etcd") - deleteAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) }) It("should scale down a single-node etcd to 0, then scale up from 0->1 replicas and then from 1->3 replicas", func() { @@ -202,9 +194,6 @@ var _ = Describe("Etcd", func() { Expect(cl.Get(ctx, client.ObjectKeyFromObject(etcd), etcd)).To(Succeed()) etcd.Spec.Replicas = 3 updateAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) - - By("Deleting the single-node etcd") - deleteAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) }) It("should scale down a single-node etcd to 0 replica, then scale up from 0->1 replica and then from 1->3 replicas with TLS enabled for cluster peerUrl", func() { @@ -232,9 +221,6 @@ var _ = Describe("Etcd", func() { etcd.Spec.Replicas = 3 etcd.Spec.Etcd.PeerUrlTLS = getPeerTls(provider.Suffix) updateAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) - - By("Deleting the single-node etcd") - deleteAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) }) }) }) @@ -266,6 +252,26 @@ func checkUnreadySts(ctx context.Context, cl client.Client, logger logr.Logger, logger.Info("sts is unready") } +func checkForUnreadyEtcdMembers(ctx context.Context, cl client.Client, logger logr.Logger, etcd *v1alpha1.Etcd, timeout time.Duration) { + logger.Info("Waiting for at least one etcd member to become unready") + EventuallyWithOffset(1, func() error { + ctx, cancelFunc := context.WithTimeout(ctx, timeout) + defer cancelFunc() + + err := cl.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: namespace}, etcd) + if err != nil { + return err + } + + if etcd.Status.ReadyReplicas == etcd.Spec.Replicas { + return fmt.Errorf("etcd ready replicas is still same as spec replicas") + } + + return nil + }, timeout, pollingInterval).Should(BeNil()) + logger.Info("at least one etcd member is unready") +} + // checkEventuallyEtcdRollingUpdateDone is a helper function, uses Gomega Eventually. // Returns the function until etcd rolling updates is done for given timeout and polling interval or // it raises assertion error. diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index a47a896b7..f5be8a0f7 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -93,6 +93,12 @@ var _ = BeforeSuite(func() { // deploy TLS secrets certsPath := path.Join(sourcePath, certsBasePath) Expect(buildAndDeployTLSSecrets(ctx, cl, logger, etcdNamespace, certsPath, providers)).To(Succeed()) + + // deploy backup secrets + storageContainer := getEnvAndExpectNoError(envStorageContainer) + for _, provider := range providers { + Expect(deployBackupSecret(ctx, cl, logger, provider, etcdNamespace, storageContainer)).To(Succeed()) + } }) var _ = AfterSuite(func() { diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index 72dbc6362..f0679a142 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -439,8 +439,7 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(0640)), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), })), }), }), @@ -496,8 +495,7 @@ func validateStoreAWSForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(0640)), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), })), }), }), @@ -553,8 +551,7 @@ func validateStoreAzureForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1 "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(0640)), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), })), }), }), @@ -610,8 +607,7 @@ func validateStoreOpenstackForCompactionJob(instance *druidv1alpha1.Etcd, j *bat "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(0640)), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), })), }), }), @@ -668,8 +664,7 @@ func validateStoreAlicloudForCompactionJob(instance *druidv1alpha1.Etcd, j *batc "Name": Equal("etcd-backup"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(0640)), + "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), })), }), }), diff --git a/test/utils/statefulset.go b/test/utils/statefulset.go index 2352f6a09..d24962b17 100644 --- a/test/utils/statefulset.go +++ b/test/utils/statefulset.go @@ -9,6 +9,8 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -50,8 +52,8 @@ func CreateStatefulSet(name, namespace string, etcdUID types.UID, replicas int32 Labels: map[string]string{ druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: name, + druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, druidv1alpha1.LabelAppNameKey: name, - druidv1alpha1.LabelComponentKey: "etcd-sts", }, Annotations: nil, OwnerReferences: []metav1.OwnerReference{{ @@ -69,17 +71,17 @@ func CreateStatefulSet(name, namespace string, etcdUID types.UID, replicas int32 Replicas: pointer.Int32(replicas), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: name, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": "etcd-statefulset", - "instance": name, - "name": "etcd", + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + druidv1alpha1.LabelPartOfKey: name, + druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelAppNameKey: name, }, }, }, From c3ca407b626e1ea32fc30afec967b6f7625fd3dc Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 5 Apr 2024 22:15:37 +0530 Subject: [PATCH 132/235] Move common constants /internal/common/constants.go --- internal/common/constants.go | 55 +++++++++ internal/controller/compaction/reconciler.go | 13 +-- .../etcdcopybackupstask/reconciler.go | 35 ++++-- .../etcdcopybackupstask/reconciler_test.go | 28 ++--- internal/operator/configmap/configmap_test.go | 3 +- internal/operator/configmap/etcdconfig.go | 8 +- internal/operator/statefulset/builder.go | 104 +++++++++--------- internal/operator/statefulset/constants.go | 37 ------- internal/operator/statefulset/statefulset.go | 5 +- internal/operator/statefulset/stsmatcher.go | 50 ++++----- internal/utils/envvar.go | 5 +- internal/utils/utils_suite_test.go | 17 --- .../controllers/compaction/reconciler_test.go | 34 +++--- .../etcdcopybackupstask/reconciler_test.go | 12 +- 14 files changed, 210 insertions(+), 196 deletions(-) delete mode 100644 internal/utils/utils_suite_test.go diff --git a/internal/common/constants.go b/internal/common/constants.go index 89f464360..4b9d1b05a 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -103,3 +103,58 @@ const ( // ChangeBackupBucketPermissionsInitContainerName is the name of the change backup bucket permissions init container. ChangeBackupBucketPermissionsInitContainerName = "change-backup-bucket-permissions" ) + +// Constants for volume names +const ( + // EtcdCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. + EtcdCAVolumeName = "etcd-ca" + // EtcdServerTLSVolumeName is the name of the volume that contains the server certificate-key pair used to set up the etcd server and etcd-wrapper HTTP server. + EtcdServerTLSVolumeName = "etcd-server-tls" + // EtcdClientTLSVolumeName is the name of the volume that contains the client certificate-key pair used by the client to communicate to the etcd server and etcd-wrapper HTTP server. + EtcdClientTLSVolumeName = "etcd-client-tls" + // EtcdPeerCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for peer communication. + EtcdPeerCAVolumeName = "etcd-peer-ca" + // EtcdPeerServerTLSVolumeName is the name of the volume that contains the server certificate-key pair used to set up the peer server. + EtcdPeerServerTLSVolumeName = "etcd-peer-server-tls" + // BackupRestoreCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for backup-restore communication. + BackupRestoreCAVolumeName = "backup-restore-ca" + // BackupRestoreServerTLSVolumeName is the name of the volume that contains the server certificate-key pair used to set up the backup-restore server. + BackupRestoreServerTLSVolumeName = "backup-restore-server-tls" + // BackupRestoreClientTLSVolumeName is the name of the volume that contains the client certificate-key pair used by the client to communicate to the backup-restore server. + BackupRestoreClientTLSVolumeName = "backup-restore-client-tls" + + // EtcdConfigVolumeName is the name of the volume that contains the etcd configuration file. + EtcdConfigVolumeName = "etcd-config-file" + // LocalBackupVolumeName is the name of the volume that contains the local backup. + LocalBackupVolumeName = "local-backup" + // ProviderBackupSecretVolumeName is the name of the volume that contains the provider backup secret. + ProviderBackupSecretVolumeName = "etcd-backup-secret" +) + +// constants for volume mount paths +const ( + // EtcdCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for client communication are mounted. + EtcdCAVolumeMountPath = "/var/etcd/ssl/ca" + // EtcdServerTLSVolumeMountPath is the path on a container where the server certificate-key pair used to set up the etcd server and etcd-wrapper HTTP server is mounted. + EtcdServerTLSVolumeMountPath = "/var/etcd/ssl/server" + // EtcdClientTLSVolumeMountPath is the path on a container where the client certificate-key pair used by the client to communicate to the etcd server and etcd-wrapper HTTP server is mounted. + EtcdClientTLSVolumeMountPath = "/var/etcd/ssl/client" + // EtcdPeerCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for peer communication are mounted. + EtcdPeerCAVolumeMountPath = "/var/etcd/ssl/peer/ca" + // EtcdPeerServerTLSVolumeMountPath is the path on a container where the server certificate-key pair used to set up the peer server is mounted. + EtcdPeerServerTLSVolumeMountPath = "/var/etcd/ssl/peer/server" + // BackupRestoreCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for backup-restore communication are mounted. + BackupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" + // BackupRestoreServerTLSVolumeMountPath is the path on a container where the server certificate-key pair used to set up the backup-restore server is mounted. + BackupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" + // BackupRestoreClientTLSVolumeMountPath is the path on a container where the client certificate-key pair used by the client to communicate to the backup-restore server is mounted. + BackupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" + + // GCSBackupVolumeMountPath is the path on a container where the GCS backup secret is mounted. + GCSBackupVolumeMountPath = "/var/.gcp/" + // NonGCSProviderBackupVolumeMountPath is the path on a container where the non-GCS provider backup secret is mounted. + NonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" + + // EtcdDataVolumeMountPath is the path on a container where the etcd data directory is hosted. + EtcdDataVolumeMountPath = "/var/etcd/data" +) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index fe54e1a5c..6470d9e50 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -15,7 +15,6 @@ import ( ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" druidmetrics "github.com/gardener/etcd-druid/internal/metrics" - "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -358,7 +357,7 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu vms := []v1.VolumeMount{ { Name: "etcd-workspace-dir", - MountPath: statefulset.EtcdDataVolumeMountPath, + MountPath: common.EtcdDataVolumeMountPath, }, } @@ -381,13 +380,13 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu } case utils.GCS: vms = append(vms, v1.VolumeMount{ - Name: "etcd-backup", - MountPath: statefulset.GCSBackupVolumeMountPath, + Name: common.ProviderBackupSecretVolumeName, + MountPath: common.GCSBackupVolumeMountPath, }) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: vms = append(vms, v1.VolumeMount{ - Name: "etcd-backup", - MountPath: statefulset.NonGCSProviderBackupVolumeMountPath, + Name: common.ProviderBackupSecretVolumeName, + MountPath: common.NonGCSProviderBackupVolumeMountPath, }) } @@ -432,7 +431,7 @@ func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr. } vs = append(vs, v1.Volume{ - Name: "etcd-backup", + Name: common.ProviderBackupSecretVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: storeValues.SecretRef.Name, diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 840f810ee..f69f67eda 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "strconv" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" @@ -459,7 +460,7 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a return } volumes = append(volumes, corev1.Volume{ - Name: getVolumeNamePrefix(prefix) + "etcd-backup", + Name: getVolumeNamePrefix(prefix) + common.ProviderBackupSecretVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, @@ -491,18 +492,30 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum } case utils.GCS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", - MountPath: "/var/." + getVolumeNamePrefix(volumeMountPrefix) + "gcp/", + Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, + MountPath: getGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), }) case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup", - MountPath: "/var/" + getVolumeNamePrefix(volumeMountPrefix) + "etcd-backup/", + Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, + MountPath: getNonGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), }) } return } +func getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { + // "/var/etcd-backup" + tokens := strings.Split(strings.Trim(common.NonGCSProviderBackupVolumeMountPath, "/"), "/") + return fmt.Sprintf("/%s/%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) +} + +func getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { + // "/var/.gcp" + tokens := strings.Split(strings.TrimSuffix(common.GCSBackupVolumeMountPath, "/"), ".") + return fmt.Sprintf("%s.%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) +} + // createEnvVarsFromStore generates a slice of environment variables for an EtcdCopyBackups job based on the given StoreSpec, // storeProvider, prefix, and volumePrefix. The prefix is used to differentiate between source and target environment variables. // This function creates the necessary environment variables for various storage providers and configurations. The generated @@ -511,17 +524,17 @@ func createEnvVarsFromStore(store *druidv1alpha1.StoreSpec, storeProvider, envKe envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvStorageContainer, *store.Container)) switch storeProvider { case utils.S3: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.ABS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.GCS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, "/var/."+volumePrefix+"gcp/serviceaccount.json")) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"))) case utils.Swift: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.OCS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.OSS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, "/var/"+volumePrefix+"etcd-backup")) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) } return envVars } diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index ccb8108f2..25aea6c06 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -454,11 +454,11 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { expectedMountName = volumeMountPrefix + "host-storage" expectedMountPath = *storeSpec.Container case utils.GCS: - expectedMountName = volumeMountPrefix + "etcd-backup" - expectedMountPath = "/var/." + volumeMountPrefix + "gcp/" + expectedMountName = volumeMountPrefix + common.ProviderBackupSecretVolumeName + expectedMountPath = getGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: - expectedMountName = volumeMountPrefix + "etcd-backup" - expectedMountPath = "/var/" + volumeMountPrefix + "etcd-backup/" + expectedMountName = volumeMountPrefix + common.ProviderBackupSecretVolumeName + expectedMountPath = getNonGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") default: Fail(fmt.Sprintf("Unknown provider: %s", provider)) } @@ -604,7 +604,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { volumes, err := reconciler.createVolumesFromStore(ctx, store, namespace, string(storageProvider), "source-") Expect(err).NotTo(HaveOccurred()) Expect(volumes).To(HaveLen(1)) - Expect(volumes[0].Name).To(Equal("source-etcd-backup")) + Expect(volumes[0].Name).To(Equal("source-etcd-backup-secret")) // Assert that the volume is created correctly with the expected secret volumeSource := volumes[0].VolumeSource @@ -690,12 +690,12 @@ func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefi case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], - Value: "/var/" + volumePrefix + "etcd-backup", + Value: getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""), }) case utils.GCS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], - Value: "/var/." + volumePrefix + "gcp/serviceaccount.json", + Value: getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"), }) } Expect(envVars).To(Equal(expected)) @@ -881,7 +881,7 @@ func getProviderEnvElements(storeProvider, prefix, volumePrefix string) Elements return Elements{ prefix + common.EnvGoogleApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(prefix + common.EnvGoogleApplicationCredentials), - "Value": Equal(fmt.Sprintf("/var/.%sgcp/serviceaccount.json", volumePrefix)), + "Value": Equal(getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json")), }), } case "Swift": @@ -914,15 +914,15 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { switch storeProvider { case "GCS": return Elements{ - volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), + volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), }), } default: return Elements{ - volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), + volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), "MountPath": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } @@ -931,8 +931,8 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Elements { return Elements{ - volumePrefix + "etcd-backup": MatchAllFields(Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), + volumePrefix + common.ProviderBackupSecretVolumeName: MatchAllFields(Fields{ + "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(store.SecretRef.Name), diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 8a584e2da..e20bd3de6 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -14,7 +14,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/component" - "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" @@ -336,7 +335,7 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C g.Expect(err).ToNot(HaveOccurred()) g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "name": Equal(fmt.Sprintf("etcd-%s", etcd.UID[:6])), - "data-dir": Equal(fmt.Sprintf("%s/new.etcd", statefulset.EtcdDataVolumeMountPath)), + "data-dir": Equal(fmt.Sprintf("%s/new.etcd", common.EtcdDataVolumeMountPath)), "metrics": Equal(string(druidv1alpha1.Basic)), "snapshot-count": Equal(int64(75000)), "enable-v2": Equal(false), diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index 89401c608..ada0b8f0d 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -10,7 +10,7 @@ import ( "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/operator/statefulset" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/utils" "k8s.io/utils/pointer" ) @@ -30,7 +30,7 @@ const ( ) var ( - defaultDataDir = fmt.Sprintf("%s/new.etcd", statefulset.EtcdDataVolumeMountPath) + defaultDataDir = fmt.Sprintf("%s/new.etcd", common.EtcdDataVolumeMountPath) ) type tlsTarget string @@ -69,8 +69,8 @@ type securityConfig struct { } func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { - clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, statefulset.EtcdCAVolumeMountPath, statefulset.EtcdServerTLSVolumeMountPath) - peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, statefulset.EtcdPeerCAVolumeMountPath, statefulset.EtcdPeerServerTLSVolumeMountPath) + clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, common.EtcdCAVolumeMountPath, common.EtcdServerTLSVolumeMountPath) + peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, common.EtcdPeerCAVolumeMountPath, common.EtcdPeerServerTLSVolumeMountPath) cfg := &etcdConfig{ Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index f2d5840c3..68b40fb80 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -222,7 +222,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 %s", EtcdDataVolumeMountPath)}, + Args: []string{fmt.Sprintf("chown -R 65532:65532 %s", common.EtcdDataVolumeMountPath)}, VolumeMounts: []corev1.VolumeMount{b.getEtcdDataVolumeMount()}, SecurityContext: &corev1.SecurityContext{ RunAsGroup: pointer.Int64(0), @@ -265,7 +265,7 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun brVolumeMounts = append(brVolumeMounts, b.getEtcdDataVolumeMount(), corev1.VolumeMount{ - Name: etcdConfigVolumeName, + Name: common.EtcdConfigVolumeName, MountPath: etcdConfigFileMountPath, }, ) @@ -285,20 +285,20 @@ func (b *stsBuilder) getBackupRestoreContainerSecretVolumeMounts() []corev1.Volu if b.etcd.Spec.Backup.TLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: backupRestoreServerTLSVolumeName, - MountPath: backupRestoreServerTLSVolumeMountPath, + Name: common.BackupRestoreServerTLSVolumeName, + MountPath: common.BackupRestoreServerTLSVolumeMountPath, }, ) } if b.etcd.Spec.Etcd.ClientUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: etcdCAVolumeName, - MountPath: EtcdCAVolumeMountPath, + Name: common.EtcdCAVolumeName, + MountPath: common.EtcdCAVolumeMountPath, }, corev1.VolumeMount{ - Name: etcdClientTLSVolumeName, - MountPath: etcdClientTLSVolumeMountPath, + Name: common.EtcdClientTLSVolumeName, + MountPath: common.EtcdClientTLSVolumeMountPath, }, ) } @@ -312,25 +312,25 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { if b.etcd.Spec.Backup.Store.Container != nil { if b.useEtcdWrapper { return &corev1.VolumeMount{ - Name: localBackupVolumeName, + Name: common.LocalBackupVolumeName, MountPath: fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, "")), } } else { return &corev1.VolumeMount{ - Name: localBackupVolumeName, + Name: common.LocalBackupVolumeName, MountPath: pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, ""), } } } case utils.GCS: return &corev1.VolumeMount{ - Name: providerBackupVolumeName, - MountPath: GCSBackupVolumeMountPath, + Name: common.ProviderBackupSecretVolumeName, + MountPath: common.GCSBackupVolumeMountPath, } case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: return &corev1.VolumeMount{ - Name: providerBackupVolumeName, - MountPath: NonGCSProviderBackupVolumeMountPath, + Name: common.ProviderBackupSecretVolumeName, + MountPath: common.NonGCSProviderBackupVolumeMountPath, } } return nil @@ -340,7 +340,7 @@ func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) return corev1.VolumeMount{ Name: volumeClaimTemplateName, - MountPath: EtcdDataVolumeMountPath, + MountPath: common.EtcdDataVolumeMountPath, } } @@ -437,9 +437,9 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { // ----------------------------------------------------------------------------------------------------------------- if b.etcd.Spec.Etcd.ClientUrlTLS != nil { dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - commandArgs = append(commandArgs, fmt.Sprintf("--cacert=%s/%s", EtcdCAVolumeMountPath, dataKey)) - commandArgs = append(commandArgs, fmt.Sprintf("--cert=%s/tls.crt", etcdClientTLSVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--key=%s/tls.key", etcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--cacert=%s/%s", common.EtcdCAVolumeMountPath, dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--cert=%s/tls.crt", common.EtcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--key=%s/tls.key", common.EtcdClientTLSVolumeMountPath)) commandArgs = append(commandArgs, "--insecure-transport=false") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=false") commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) @@ -451,15 +451,15 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) } if b.etcd.Spec.Backup.TLS != nil { - commandArgs = append(commandArgs, fmt.Sprintf("--server-cert=%s/tls.crt", backupRestoreServerTLSVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--server-key=%s/tls.key", backupRestoreServerTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--server-cert=%s/tls.crt", common.BackupRestoreServerTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--server-key=%s/tls.key", common.BackupRestoreServerTLSVolumeMountPath)) } // Other misc command line args // ----------------------------------------------------------------------------------------------------------------- - commandArgs = append(commandArgs, fmt.Sprintf("--data-dir=%s/new.etcd", EtcdDataVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--restoration-temp-snapshots-dir=%s/restoration.temp", EtcdDataVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--snapstore-temp-directory=%s/temp", EtcdDataVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--data-dir=%s/new.etcd", common.EtcdDataVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--restoration-temp-snapshots-dir=%s/restoration.temp", common.EtcdDataVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--snapstore-temp-directory=%s/temp", common.EtcdDataVolumeMountPath)) commandArgs = append(commandArgs, fmt.Sprintf("--etcd-connection-timeout=%s", defaultEtcdConnectionTimeout)) commandArgs = append(commandArgs, "--enable-member-lease-renewal=true") @@ -589,9 +589,9 @@ func (b *stsBuilder) getEtcdContainerReadinessProbeCommand() []string { cmdBuilder.WriteString("ETCDCTL_API=3 etcdctl") if b.etcd.Spec.Etcd.ClientUrlTLS != nil { dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - cmdBuilder.WriteString(fmt.Sprintf(" --cacert=%s/%s", EtcdCAVolumeMountPath, dataKey)) - cmdBuilder.WriteString(fmt.Sprintf(" --cert=%s/tls.crt", etcdClientTLSVolumeMountPath)) - cmdBuilder.WriteString(fmt.Sprintf(" --key=%s/tls.key", etcdClientTLSVolumeMountPath)) + cmdBuilder.WriteString(fmt.Sprintf(" --cacert=%s/%s", common.EtcdCAVolumeMountPath, dataKey)) + cmdBuilder.WriteString(fmt.Sprintf(" --cert=%s/tls.crt", common.EtcdClientTLSVolumeMountPath)) + cmdBuilder.WriteString(fmt.Sprintf(" --key=%s/tls.key", common.EtcdClientTLSVolumeMountPath)) cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) } else { cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", b.etcd.Name, b.clientPort)) @@ -620,9 +620,9 @@ func (b *stsBuilder) getEtcdContainerCommandArgs() []string { } else { commandArgs = append(commandArgs, "--backup-restore-tls-enabled=true") dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=%s/%s", backupRestoreCAVolumeMountPath, dataKey)) - commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-cert-path=%s/tls.crt", etcdClientTLSVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-key-path=%s/tls.key", etcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=%s/%s", common.BackupRestoreCAVolumeMountPath, dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-cert-path=%s/tls.crt", common.EtcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-key-path=%s/tls.key", common.EtcdClientTLSVolumeMountPath)) } return commandArgs } @@ -658,36 +658,36 @@ func (b *stsBuilder) getEtcdContainerSecretVolumeMounts() []corev1.VolumeMount { if b.etcd.Spec.Etcd.ClientUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: etcdCAVolumeName, - MountPath: EtcdCAVolumeMountPath, + Name: common.EtcdCAVolumeName, + MountPath: common.EtcdCAVolumeMountPath, }, corev1.VolumeMount{ - Name: etcdServerTLSVolumeName, - MountPath: EtcdServerTLSVolumeMountPath, + Name: common.EtcdServerTLSVolumeName, + MountPath: common.EtcdServerTLSVolumeMountPath, }, corev1.VolumeMount{ - Name: etcdClientTLSVolumeName, - MountPath: etcdClientTLSVolumeMountPath, + Name: common.EtcdClientTLSVolumeName, + MountPath: common.EtcdClientTLSVolumeMountPath, }, ) } if b.etcd.Spec.Etcd.PeerUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: etcdPeerCAVolumeName, - MountPath: EtcdPeerCAVolumeMountPath, + Name: common.EtcdPeerCAVolumeName, + MountPath: common.EtcdPeerCAVolumeMountPath, }, corev1.VolumeMount{ - Name: etcdPeerServerTLSVolumeName, - MountPath: EtcdPeerServerTLSVolumeMountPath, + Name: common.EtcdPeerServerTLSVolumeName, + MountPath: common.EtcdPeerServerTLSVolumeMountPath, }, ) } if b.etcd.Spec.Backup.TLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: backupRestoreCAVolumeName, - MountPath: backupRestoreCAVolumeMountPath, + Name: common.BackupRestoreCAVolumeName, + MountPath: common.BackupRestoreCAVolumeMountPath, }, ) } @@ -698,7 +698,7 @@ func (b *stsBuilder) getEtcdContainerSecretVolumeMounts() []corev1.VolumeMount { func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volume, error) { volumes := []corev1.Volume{ { - Name: etcdConfigVolumeName, + Name: common.EtcdConfigVolumeName, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ @@ -741,7 +741,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { clientTLSConfig := b.etcd.Spec.Etcd.ClientUrlTLS return []corev1.Volume{ { - Name: etcdCAVolumeName, + Name: common.EtcdCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.TLSCASecretRef.Name, @@ -750,7 +750,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { }, }, { - Name: etcdServerTLSVolumeName, + Name: common.EtcdServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ServerTLSSecretRef.Name, @@ -759,7 +759,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { }, }, { - Name: etcdClientTLSVolumeName, + Name: common.EtcdClientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ClientTLSSecretRef.Name, @@ -774,7 +774,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { peerTLSConfig := b.etcd.Spec.Etcd.PeerUrlTLS return []corev1.Volume{ { - Name: etcdPeerCAVolumeName, + Name: common.EtcdPeerCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.TLSCASecretRef.Name, @@ -783,7 +783,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { }, }, { - Name: etcdPeerServerTLSVolumeName, + Name: common.EtcdPeerServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.ServerTLSSecretRef.Name, @@ -798,7 +798,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { tlsConfig := b.etcd.Spec.Backup.TLS return []corev1.Volume{ { - Name: backupRestoreCAVolumeName, + Name: common.BackupRestoreCAVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.TLSCASecretRef.Name, @@ -807,7 +807,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { }, }, { - Name: backupRestoreServerTLSVolumeName, + Name: common.BackupRestoreServerTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ServerTLSSecretRef.Name, @@ -816,7 +816,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { }, }, { - Name: backupRestoreClientTLSVolumeName, + Name: common.BackupRestoreClientTLSVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ClientTLSSecretRef.Name, @@ -841,7 +841,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol hpt := corev1.HostPathDirectory return &corev1.Volume{ - Name: localBackupVolumeName, + Name: common.LocalBackupVolumeName, VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: hostPath + "/" + pointer.StringDeref(store.Container, ""), @@ -855,7 +855,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol } return &corev1.Volume{ - Name: providerBackupVolumeName, + Name: common.ProviderBackupSecretVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, diff --git a/internal/operator/statefulset/constants.go b/internal/operator/statefulset/constants.go index 21f743c63..6a423421f 100644 --- a/internal/operator/statefulset/constants.go +++ b/internal/operator/statefulset/constants.go @@ -4,48 +4,11 @@ package statefulset -// constants for volume names -const ( - // etcdCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. - etcdCAVolumeName = "etcd-ca" - // etcdServerTLSVolumeName is the name of the volume that contains the server certificate used to set up the server (etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). - etcdServerTLSVolumeName = "etcd-server-tls" - // etcdClientTLSVolumeName is the name of the volume that contains the client certificate used by the client to communicate to the server(etcd server, etcd-backup-restore HTTP server and etcd-wrapper HTTP server). - etcdClientTLSVolumeName = "etcd-client-tls" - etcdPeerCAVolumeName = "etcd-peer-ca" - etcdPeerServerTLSVolumeName = "etcd-peer-server-tls" - backupRestoreCAVolumeName = "backup-restore-ca" - backupRestoreServerTLSVolumeName = "backup-restore-server-tls" - backupRestoreClientTLSVolumeName = "backup-restore-client-tls" - etcdConfigVolumeName = "etcd-config-file" - localBackupVolumeName = "local-backup" - providerBackupVolumeName = "etcd-backup" -) - -// constants for volume mount paths -const ( - EtcdCAVolumeMountPath = "/var/etcd/ssl/ca" - EtcdServerTLSVolumeMountPath = "/var/etcd/ssl/server" - etcdClientTLSVolumeMountPath = "/var/etcd/ssl/client" - EtcdPeerCAVolumeMountPath = "/var/etcd/ssl/peer/ca" - EtcdPeerServerTLSVolumeMountPath = "/var/etcd/ssl/peer/server" - backupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" - backupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" - backupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" - GCSBackupVolumeMountPath = "/var/.gcp/" - NonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" -) - const ( etcdConfigFileName = "etcd.conf.yaml" etcdConfigFileMountPath = "/var/etcd/config/" ) -const ( - // EtcdDataVolumeMountPath is the path on etcd and etcd-backup-restore containers where etcd data directory is hosted. - EtcdDataVolumeMountPath = "/var/etcd/data" -) - // constants for container ports const ( serverPortName = "server" diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index b882141bf..4c72f94ed 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -8,6 +8,7 @@ import ( "fmt" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/operator/component" @@ -182,10 +183,10 @@ func isStatefulSetPatchedWithPeerTLSVolMount(existingSts *appsv1.StatefulSet) bo volumes := existingSts.Spec.Template.Spec.Volumes var peerURLCAEtcdVolPresent, peerURLEtcdServerTLSVolPresent bool for _, vol := range volumes { - if vol.Name == etcdPeerCAVolumeName { + if vol.Name == common.EtcdPeerCAVolumeName { peerURLCAEtcdVolPresent = true } - if vol.Name == etcdPeerServerTLSVolumeName { + if vol.Name == common.EtcdPeerServerTLSVolumeName { peerURLEtcdServerTLSVolPresent = true } } diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 7a853cf42..069b16e8c 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -325,13 +325,13 @@ func (s StatefulSetMatcher) matchEtcdContainerCmdArgs() gomegatypes.GomegaMatche func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) - return matchVolMount(volumeClaimTemplateName, EtcdDataVolumeMountPath) + return matchVolMount(volumeClaimTemplateName, common.EtcdDataVolumeMountPath) } func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.GomegaMatcher { volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) - volMountMatchers = append(volMountMatchers, matchVolMount(etcdConfigVolumeName, etcdConfigFileMountPath)) + volMountMatchers = append(volMountMatchers, matchVolMount(common.EtcdConfigVolumeName, etcdConfigFileMountPath)) volMountMatchers = append(volMountMatchers, s.getBackupRestoreSecretVolMountMatchers()...) if s.etcd.IsBackupStoreEnabled() { etcdBackupVolMountMatcher := s.getEtcdBackupVolumeMountMatcher() @@ -345,9 +345,9 @@ func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.G func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 3) if s.etcd.Spec.Backup.TLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backupRestoreCAVolumeName, backupRestoreCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backupRestoreServerTLSVolumeName, backupRestoreServerTLSVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(backupRestoreClientTLSVolumeName, backupRestoreClientTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.BackupRestoreCAVolumeName, common.BackupRestoreCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.BackupRestoreServerTLSVolumeName, common.BackupRestoreServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.BackupRestoreClientTLSVolumeName, common.BackupRestoreClientTLSVolumeMountPath)) } return secretVolMountMatchers } @@ -355,13 +355,13 @@ func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatyp func (s StatefulSetMatcher) getEtcdSecretVolMountsMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdCAVolumeName, EtcdCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdServerTLSVolumeName, EtcdServerTLSVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdClientTLSVolumeName, etcdClientTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdCAVolumeName, common.EtcdCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdServerTLSVolumeName, common.EtcdServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdClientTLSVolumeName, common.EtcdClientTLSVolumeMountPath)) } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdPeerCAVolumeName, EtcdPeerCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(etcdPeerServerTLSVolumeName, EtcdPeerServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdPeerCAVolumeName, common.EtcdPeerCAVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdPeerServerTLSVolumeName, common.EtcdPeerServerTLSVolumeMountPath)) } return secretVolMountMatchers } @@ -371,15 +371,15 @@ func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.Gomega case utils.Local: if s.etcd.Spec.Backup.Store.Container != nil { if s.useEtcdWrapper { - return matchVolMount(localBackupVolumeName, fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))) + return matchVolMount(common.LocalBackupVolumeName, fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))) } else { - return matchVolMount(localBackupVolumeName, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) + return matchVolMount(common.LocalBackupVolumeName, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) } } case utils.GCS: - return matchVolMount(providerBackupVolumeName, GCSBackupVolumeMountPath) + return matchVolMount(common.ProviderBackupSecretVolumeName, common.GCSBackupVolumeMountPath) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - return matchVolMount(providerBackupVolumeName, NonGCSProviderBackupVolumeMountPath) + return matchVolMount(common.ProviderBackupSecretVolumeName, common.NonGCSProviderBackupVolumeMountPath) } return nil } @@ -416,7 +416,7 @@ func (s StatefulSetMatcher) matchEtcdPodSecurityContext() gomegatypes.GomegaMatc func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { volMatchers := make([]gomegatypes.GomegaMatcher, 0, 7) etcdConfigFileVolMountMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcdConfigVolumeName), + "Name": Equal(common.EtcdConfigVolumeName), "VolumeSource": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ @@ -448,7 +448,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM volMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcdCAVolumeName), + "Name": Equal(common.EtcdCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), @@ -457,7 +457,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcdServerTLSVolumeName), + "Name": Equal(common.EtcdServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), @@ -466,7 +466,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcdClientTLSVolumeName), + "Name": Equal(common.EtcdClientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), @@ -477,7 +477,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcdPeerCAVolumeName), + "Name": Equal(common.EtcdPeerCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), @@ -487,7 +487,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcdPeerServerTLSVolumeName), + "Name": Equal(common.EtcdPeerServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), @@ -498,7 +498,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM } if s.etcd.Spec.Backup.TLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(backupRestoreCAVolumeName), + "Name": Equal(common.BackupRestoreCAVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), @@ -508,7 +508,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(backupRestoreServerTLSVolumeName), + "Name": Equal(common.BackupRestoreServerTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), @@ -518,7 +518,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(backupRestoreClientTLSVolumeName), + "Name": Equal(common.BackupRestoreClientTLSVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), @@ -539,7 +539,7 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { hostPath, err := utils.GetHostMountPathFromSecretRef(context.Background(), s.cl, logr.Discard(), s.etcd.Spec.Backup.Store, s.etcd.Namespace) s.g.Expect(err).ToNot(HaveOccurred()) return MatchFields(IgnoreExtras, Fields{ - "Name": Equal(localBackupVolumeName), + "Name": Equal(common.LocalBackupVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "HostPath": PointTo(MatchFields(IgnoreExtras, Fields{ "Path": Equal(fmt.Sprintf("%s/%s", hostPath, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))), @@ -551,7 +551,7 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: s.g.Expect(s.etcd.Spec.Backup.Store.SecretRef).ToNot(BeNil()) return MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), + "Name": Equal(common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.Store.SecretRef.Name), diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index dde1f2aee..2a0caeb9e 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -6,6 +6,7 @@ package utils import ( "fmt" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" @@ -59,7 +60,7 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) return nil, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") } - const credentialsMountPath = "/var/etcd-backup" + credentialsMountPath := strings.TrimSuffix(common.NonGCSProviderBackupVolumeMountPath, "/") switch provider { case S3: envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, credentialsMountPath)) @@ -68,7 +69,7 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, credentialsMountPath)) case GCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, "/var/.gcp/serviceaccount.json")) + envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.GCSBackupVolumeMountPath))) envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) case Swift: diff --git a/internal/utils/utils_suite_test.go b/internal/utils/utils_suite_test.go deleted file mode 100644 index 7faf50c41..000000000 --- a/internal/utils/utils_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package utils_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestUtils(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Utils Suite") -} diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index f0679a142..f8f0dad36 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -389,9 +389,9 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container): Equal(fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container)), }), "VolumeMounts": MatchElements(testutils.VolumeMountIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), - "MountPath": Equal("/var/.gcp/"), + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.ProviderBackupSecretVolumeName), + "MountPath": Equal(common.GCSBackupVolumeMountPath), }), }), "Env": MatchAllElements(testutils.EnvIterator, Elements{ @@ -435,8 +435,8 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -485,14 +485,14 @@ func validateStoreAWSForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J }), common.EnvAWSApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAWSApplicationCredentials), - "Value": Equal("/var/etcd-backup"), + "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -541,14 +541,14 @@ func validateStoreAzureForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1 }), common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAzureApplicationCredentials), - "Value": Equal("/var/etcd-backup"), + "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -597,14 +597,14 @@ func validateStoreOpenstackForCompactionJob(instance *druidv1alpha1.Etcd, j *bat }), common.EnvOpenstackApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvOpenstackApplicationCredentials), - "Value": Equal("/var/etcd-backup"), + "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -654,14 +654,14 @@ func validateStoreAlicloudForCompactionJob(instance *druidv1alpha1.Etcd, j *batc }), common.EnvAlicloudApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAlicloudApplicationCredentials), - "Value": Equal("/var/etcd-backup"), + "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-backup"), + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index c049e497c..198c14bdb 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -334,15 +334,15 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { switch storeProvider { case "GCS": return Elements{ - volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), + volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), }), } default: return Elements{ - volumePrefix + "etcd-backup": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), + volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), "MountPath": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } @@ -352,8 +352,8 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Elements { return Elements{ - volumePrefix + "etcd-backup": MatchAllFields(Fields{ - "Name": Equal(volumePrefix + "etcd-backup"), + volumePrefix + common.ProviderBackupSecretVolumeName: MatchAllFields(Fields{ + "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(store.SecretRef.Name), From b40fa1596a79ffbfc5eafef15855f9574fb8c25d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 17 Apr 2024 11:31:54 +0530 Subject: [PATCH 133/235] moved back to go 1.21 and fixed unit test for etcdcopybackup controller --- go.mod | 2 +- internal/controller/etcdcopybackupstask/reconciler.go | 2 +- internal/controller/etcdcopybackupstask/reconciler_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4e3377c6b..65433bdd5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gardener/etcd-druid -go 1.22 +go 1.21 require ( github.com/gardener/etcd-backup-restore v0.26.0 diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index f69f67eda..cbe256143 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -493,7 +493,7 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum case utils.GCS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, - MountPath: getGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), + MountPath: getGCSVolumeMountPathWithPrefixAndSuffix(getVolumeNamePrefix(volumeMountPrefix), "/"), }) case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 25aea6c06..c49a16bb9 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -915,7 +915,7 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { case "GCS": return Elements{ volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), + "Name": Equal(getVolumeNamePrefix(volumePrefix) + common.ProviderBackupSecretVolumeName), "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), }), } From 0a28a1ee3772c1caba8b21ee2ac75b33b18b737d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 17 Apr 2024 11:34:03 +0530 Subject: [PATCH 134/235] changed version of go to 1.21.4 in dockerfile and pipeline_definition --- .ci/pipeline_definitions | 8 ++++---- Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.ci/pipeline_definitions b/.ci/pipeline_definitions index 25c458a74..71f5a990d 100644 --- a/.ci/pipeline_definitions +++ b/.ci/pipeline_definitions @@ -35,13 +35,13 @@ etcd-druid: teamname: 'gardener/etcd-druid-maintainers' steps: check: - image: 'golang:1.22.1' + image: 'golang:1.21.4' test: - image: 'golang:1.22.1' + image: 'golang:1.21.4' test_integration: - image: 'golang:1.22.1' + image: 'golang:1.21.4' build: - image: 'golang:1.22.1' + image: 'golang:1.21.4' output_dir: 'binary' jobs: diff --git a/Dockerfile b/Dockerfile index 767974bd9..94a9e3178 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22.0 as builder +FROM golang:1.21.4 as builder WORKDIR /go/src/github.com/gardener/etcd-druid COPY . . From 1f99b238cb0d8fa8af972a8d4e3c0e28c3baa62c Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 18 Apr 2024 16:31:20 +0530 Subject: [PATCH 135/235] rebased with upstream master, resolved compilation issues, fixed make check and make test --- Makefile | 10 +- api/v1alpha1/types_etcd_test.go | 18 +- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 27 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 1309 +++--- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 27 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 3599 ++++++++--------- go.mod | 2 +- internal/common/constants.go | 6 +- internal/controller/etcd/register_test.go | 2 +- .../etcdcopybackupstask/reconciler_test.go | 39 +- internal/controller/manager.go | 21 +- internal/controller/secret/register.go | 4 +- internal/controller/utils/reconciler.go | 15 +- .../condition/check_backup_ready_test.go | 2 +- .../check_data_volumes_ready_test.go | 2 +- .../health/etcdmember/check_ready_test.go | 2 +- .../mock/controller-runtime/client/doc.go | 1 + .../mock/controller-runtime/client/mocks.go | 33 +- .../mock/controller-runtime/manager/mocks.go | 54 +- internal/utils/envvar.go | 12 +- internal/webhook/sentinel/handler.go | 6 +- internal/webhook/sentinel/handler_test.go | 57 +- test/it/controller/assets/assets.go | 2 +- test/it/setup/setup.go | 28 +- test/utils/client.go | 8 + vendor/cloud.google.com/go/.gitignore | 12 - .../.release-please-manifest-submodules.json | 113 - .../go/.release-please-manifest.json | 3 - vendor/cloud.google.com/go/CHANGES.md | 2424 ----------- vendor/cloud.google.com/go/CODE_OF_CONDUCT.md | 44 - vendor/cloud.google.com/go/CONTRIBUTING.md | 327 -- vendor/cloud.google.com/go/LICENSE | 202 - vendor/cloud.google.com/go/README.md | 139 - vendor/cloud.google.com/go/RELEASING.md | 141 - vendor/cloud.google.com/go/SECURITY.md | 7 - vendor/cloud.google.com/go/compute/LICENSE | 202 - .../go/compute/internal/version.go | 18 - .../go/compute/metadata/CHANGES.md | 5 - .../go/compute/metadata/LICENSE | 202 - .../go/compute/metadata/README.md | 27 - .../go/compute/metadata/metadata.go | 542 --- .../go/compute/metadata/retry.go | 114 - .../go/compute/metadata/retry_linux.go | 26 - .../go/compute/metadata/tidyfix.go | 23 - vendor/cloud.google.com/go/doc.go | 248 -- vendor/cloud.google.com/go/iam/CHANGES.md | 62 - vendor/cloud.google.com/go/iam/LICENSE | 202 - vendor/cloud.google.com/go/iam/README.md | 40 - vendor/cloud.google.com/go/iam/iam.go | 387 -- .../go/internal/.repo-metadata-full.json | 1946 --------- vendor/cloud.google.com/go/internal/README.md | 18 - .../cloud.google.com/go/internal/annotate.go | 55 - .../go/internal/optional/optional.go | 108 - vendor/cloud.google.com/go/internal/retry.go | 85 - .../go/internal/trace/trace.go | 111 - .../go/internal/version/update_version.sh | 19 - .../go/internal/version/version.go | 71 - vendor/cloud.google.com/go/migration.md | 50 - ...elease-please-config-yoshi-submodules.json | 341 -- .../go/release-please-config.json | 11 - vendor/cloud.google.com/go/testing.md | 236 -- .../github.com/golang-jwt/jwt/v5/.gitignore | 4 - vendor/github.com/golang-jwt/jwt/v5/LICENSE | 9 - .../golang-jwt/jwt/v5/MIGRATION_GUIDE.md | 195 - vendor/github.com/golang-jwt/jwt/v5/README.md | 167 - .../github.com/golang-jwt/jwt/v5/SECURITY.md | 19 - .../golang-jwt/jwt/v5/VERSION_HISTORY.md | 137 - vendor/github.com/golang-jwt/jwt/v5/claims.go | 16 - vendor/github.com/golang-jwt/jwt/v5/doc.go | 4 - vendor/github.com/golang-jwt/jwt/v5/ecdsa.go | 134 - .../golang-jwt/jwt/v5/ecdsa_utils.go | 69 - .../github.com/golang-jwt/jwt/v5/ed25519.go | 79 - .../golang-jwt/jwt/v5/ed25519_utils.go | 64 - vendor/github.com/golang-jwt/jwt/v5/errors.go | 49 - .../golang-jwt/jwt/v5/errors_go1_20.go | 47 - .../golang-jwt/jwt/v5/errors_go_other.go | 78 - vendor/github.com/golang-jwt/jwt/v5/hmac.go | 104 - .../golang-jwt/jwt/v5/map_claims.go | 109 - vendor/github.com/golang-jwt/jwt/v5/none.go | 50 - vendor/github.com/golang-jwt/jwt/v5/parser.go | 238 -- .../golang-jwt/jwt/v5/parser_option.go | 128 - .../golang-jwt/jwt/v5/registered_claims.go | 63 - vendor/github.com/golang-jwt/jwt/v5/rsa.go | 93 - .../github.com/golang-jwt/jwt/v5/rsa_pss.go | 135 - .../github.com/golang-jwt/jwt/v5/rsa_utils.go | 107 - .../golang-jwt/jwt/v5/signing_method.go | 49 - .../golang-jwt/jwt/v5/staticcheck.conf | 1 - vendor/github.com/golang-jwt/jwt/v5/token.go | 100 - .../golang-jwt/jwt/v5/token_option.go | 5 - vendor/github.com/golang-jwt/jwt/v5/types.go | 149 - .../github.com/golang-jwt/jwt/v5/validator.go | 316 -- 91 files changed, 2624 insertions(+), 13941 deletions(-) delete mode 100644 vendor/cloud.google.com/go/.gitignore delete mode 100644 vendor/cloud.google.com/go/.release-please-manifest-submodules.json delete mode 100644 vendor/cloud.google.com/go/.release-please-manifest.json delete mode 100644 vendor/cloud.google.com/go/CHANGES.md delete mode 100644 vendor/cloud.google.com/go/CODE_OF_CONDUCT.md delete mode 100644 vendor/cloud.google.com/go/CONTRIBUTING.md delete mode 100644 vendor/cloud.google.com/go/LICENSE delete mode 100644 vendor/cloud.google.com/go/README.md delete mode 100644 vendor/cloud.google.com/go/RELEASING.md delete mode 100644 vendor/cloud.google.com/go/SECURITY.md delete mode 100644 vendor/cloud.google.com/go/compute/LICENSE delete mode 100644 vendor/cloud.google.com/go/compute/internal/version.go delete mode 100644 vendor/cloud.google.com/go/compute/metadata/CHANGES.md delete mode 100644 vendor/cloud.google.com/go/compute/metadata/LICENSE delete mode 100644 vendor/cloud.google.com/go/compute/metadata/README.md delete mode 100644 vendor/cloud.google.com/go/compute/metadata/metadata.go delete mode 100644 vendor/cloud.google.com/go/compute/metadata/retry.go delete mode 100644 vendor/cloud.google.com/go/compute/metadata/retry_linux.go delete mode 100644 vendor/cloud.google.com/go/compute/metadata/tidyfix.go delete mode 100644 vendor/cloud.google.com/go/doc.go delete mode 100644 vendor/cloud.google.com/go/iam/CHANGES.md delete mode 100644 vendor/cloud.google.com/go/iam/LICENSE delete mode 100644 vendor/cloud.google.com/go/iam/README.md delete mode 100644 vendor/cloud.google.com/go/iam/iam.go delete mode 100644 vendor/cloud.google.com/go/internal/.repo-metadata-full.json delete mode 100644 vendor/cloud.google.com/go/internal/README.md delete mode 100644 vendor/cloud.google.com/go/internal/annotate.go delete mode 100644 vendor/cloud.google.com/go/internal/optional/optional.go delete mode 100644 vendor/cloud.google.com/go/internal/retry.go delete mode 100644 vendor/cloud.google.com/go/internal/trace/trace.go delete mode 100644 vendor/cloud.google.com/go/internal/version/update_version.sh delete mode 100644 vendor/cloud.google.com/go/internal/version/version.go delete mode 100644 vendor/cloud.google.com/go/migration.md delete mode 100644 vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json delete mode 100644 vendor/cloud.google.com/go/release-please-config.json delete mode 100644 vendor/cloud.google.com/go/testing.md delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/.gitignore delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/LICENSE delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/README.md delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/SECURITY.md delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/claims.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/doc.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/ecdsa.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/ed25519.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/errors.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/hmac.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/map_claims.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/none.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/parser.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/parser_option.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/registered_claims.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/rsa.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/signing_method.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/token.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/token_option.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/types.go delete mode 100644 vendor/github.com/golang-jwt/jwt/v5/validator.go diff --git a/Makefile b/Makefile index 56a02ce20..b715eeb06 100644 --- a/Makefile +++ b/Makefile @@ -75,12 +75,12 @@ fmt: @env GO111MODULE=on go fmt ./... clean: - @"$(REPO_ROOT)/vendor/github.com/gardener/gardener/hack/clean.sh" ./api/... ./internal/... + @bash $(GARDENER_HACK_DIR)/clean.sh ./api/... ./internal/... # Check packages .PHONY: check -check: $(GOLANGCI_LINT) $(GOIMPORTS) set-permissions fmt manifests - @"$(REPO_ROOT)/vendor/github.com/gardener/gardener/hack/check.sh" --golangci-lint-config=./.golangci.yaml ./api/... ./internal/... +check: $(GOLANGCI_LINT) $(GOIMPORTS) fmt manifests + @REPO_ROOT=$(REPO_ROOT) bash $(GARDENER_HACK_DIR)/check.sh --golangci-lint-config=./.golangci.yaml ./api/... ./internal/... .PHONY: check-generate check-generate: @@ -88,7 +88,7 @@ check-generate: # Generate code .PHONY: generate -generate: set-permissions manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) +generate: manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) @go generate "$(REPO_ROOT)/internal/..." @"$(REPO_ROOT)/hack/update-codegen.sh" @@ -132,7 +132,7 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) @bash $(HACK_DIR)/e2e-test/run-e2e-test.sh $(PROVIDERS) .PHONY: test-integration -test-integration: set-permissions $(GINKGO) $(SETUP_ENVTEST) +test-integration: $(GINKGO) $(SETUP_ENVTEST) @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test.sh" ./test/integration/... @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test-go.sh" ./test/it/... diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 8d1541e05..afa11f07e 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -7,8 +7,6 @@ package v1alpha1_test import ( "time" - testutils "github.com/gardener/etcd-druid/test/utils" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -356,12 +354,12 @@ func getEtcd(name, namespace string) *Etcd { Resources: &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("2Gi"), + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("2Gi"), }, Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("23m"), - "memory": testutils.ParseQuantity("128Mi"), + "cpu": resource.MustParse("23m"), + "memory": resource.MustParse("128Mi"), }, }, Store: &StoreSpec{ @@ -380,12 +378,12 @@ func getEtcd(name, namespace string) *Etcd { DefragmentationSchedule: &defragSchedule, Resources: &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("2500m"), - "memory": testutils.ParseQuantity("4Gi"), + "cpu": resource.MustParse("2500m"), + "memory": resource.MustParse("4Gi"), }, Requests: corev1.ResourceList{ - "cpu": testutils.ParseQuantity("500m"), - "memory": testutils.ParseQuantity("1000Mi"), + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("1000Mi"), }, }, ClientPort: &clientPort, diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 9aa0b085e..16951ef91 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -25,19 +25,14 @@ spec: source to a target store. properties: apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -46,9 +41,9 @@ spec: backups task. properties: maxBackupAge: - description: |- - MaxBackupAge is the maximum age in days that a backup must have in order to be copied. - By default all backups will be copied. + description: MaxBackupAge is the maximum age in days that a backup + must have in order to be copied. By default all backups will be + copied. format: int32 type: integer maxBackups: @@ -127,8 +122,8 @@ spec: snapshot before copying backups. type: boolean timeout: - description: |- - Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups + description: Timeout is the timeout for waiting for a final full + snapshot. When this timeout expires, the copying of backups will be performed anyway. No timeout or 0 means wait forever. type: string required: diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index 05a339d99..8bb938ba7 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -48,19 +48,14 @@ spec: description: Etcd is the Schema for the etcds API properties: apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -76,29 +71,23 @@ spec: and delta snapshots of etcd. properties: compactionResources: - description: |- - CompactionResources defines compute Resources required by compaction job. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + description: 'CompactionResources defines compute Resources required + by compaction job. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. type: string required: - name @@ -114,9 +103,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -125,19 +113,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true -<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' -======= - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ->>>>>>> 8571402f (Fix and run `make generate`) type: object type: object compression: @@ -168,9 +148,10 @@ spec: delta snapshots will be taken type: string deltaSnapshotRetentionPeriod: - description: |- - DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set. - The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d') + description: DeltaSnapshotRetentionPeriod defines the duration + for which delta snapshots will be retained, excluding the latest + snapshot set. The value should be a string formatted as a duration + (e.g., '1s', '2m', '3h', '4d') pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ type: string enableProfiling: @@ -212,41 +193,29 @@ spec: leadership status of corresponding etcd is checked. type: string type: object - maxBackupsLimitBasedGC: - description: |- - MaxBackupsLimitBasedGC defines the maximum number of Full snapshots to retain in Limit Based GarbageCollectionPolicy - All full snapshots beyond this limit will be garbage collected. - format: int32 - type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. format: int32 type: integer resources: - description: |- - Resources defines compute Resources required by backup-restore container. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + description: 'Resources defines compute Resources required by + backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. type: string required: - name @@ -262,9 +231,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -273,19 +241,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true -<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' -======= - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ->>>>>>> 8571402f (Fix and run `make generate`) type: object type: object store: @@ -323,9 +283,8 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -338,9 +297,8 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -378,9 +336,8 @@ spec: description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. It + has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -417,9 +374,8 @@ spec: TLS secrets for client communication to ETCD cluster properties: clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -432,9 +388,8 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -490,14 +445,14 @@ spec: - extensive type: string peerUrlTls: - description: |- - PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster - Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster. + description: PeerUrlTLS contains the ca and server TLS secrets + for peer communication within ETCD cluster Currently, PeerUrlTLS + does not require client TLS secrets for gardener implementation + of ETCD cluster. properties: clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -510,9 +465,8 @@ spec: type: object x-kubernetes-map-type: atomic serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace properties: name: description: name is unique within a namespace to reference @@ -553,29 +507,23 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resources: - description: |- - Resources defines the compute Resources required by etcd container. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + description: 'Resources defines the compute Resources required + by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. type: string required: - name @@ -591,9 +539,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -602,19 +549,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true -<<<<<<< HEAD description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' -======= - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ->>>>>>> 8571402f (Fix and run `make generate`) type: object type: object serverPort: @@ -633,35 +572,35 @@ spec: format: int32 type: integer schedulingConstraints: - description: |- - SchedulingConstraints defines the different scheduling constraints that must be applied to the - pod spec in the etcd statefulset. + description: SchedulingConstraints defines the different scheduling + constraints that must be applied to the pod spec in the etcd statefulset. Currently supported constraints are Affinity and TopologySpreadConstraints. properties: affinity: - description: |- - Affinity defines the various affinity and anti-affinity rules for a pod - that are honoured by the kube-scheduler. + description: Affinity defines the various affinity and anti-affinity + rules for a pod that are honoured by the kube-scheduler. properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). properties: preference: description: A node selector term, associated with @@ -671,26 +610,32 @@ spec: description: A list of node selector requirements by node's labels. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -703,26 +648,32 @@ spec: description: A list of node selector requirements by node's fields. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -744,46 +695,53 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -796,26 +754,32 @@ spec: description: A list of node selector requirements by node's fields. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -838,16 +802,18 @@ spec: other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred @@ -866,25 +832,30 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -896,45 +867,53 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -946,37 +925,42 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". items: type: string type: array topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: @@ -985,22 +969,23 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, @@ -1011,24 +996,28 @@ spec: selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic merge patch. items: type: string @@ -1041,44 +1030,51 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic merge patch. items: type: string @@ -1091,29 +1087,33 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey @@ -1126,16 +1126,18 @@ spec: as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node that + violates one or more of the expressions. The node that + is most preferred is the one with the greatest sum of + weights, i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + anti-affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred @@ -1154,25 +1156,30 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -1184,45 +1191,53 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. items: type: string type: array @@ -1234,37 +1249,42 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". items: type: string type: array topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: @@ -1273,22 +1293,23 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the pod + will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod + label update), the system may or may not try to eventually + evict the pod from its node. When there are multiple + elements, the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, @@ -1299,24 +1320,28 @@ spec: selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic merge patch. items: type: string @@ -1329,44 +1354,51 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic merge patch. items: type: string @@ -1379,29 +1411,33 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". items: type: string type: array topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey @@ -1410,25 +1446,24 @@ spec: type: object type: object topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology domains, - that are honoured by the kube-scheduler. + description: TopologySpreadConstraints describes how a group of + pods ought to spread across topology domains, that are honoured + by the kube-scheduler. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates the key and values. properties: key: @@ -1436,16 +1471,17 @@ spec: applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. items: type: string type: array @@ -1457,15 +1493,15 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: -<<<<<<< HEAD description: "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod @@ -1479,123 +1515,110 @@ spec: against labelSelector. \n This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default)." -======= - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. ->>>>>>> 8571402f (Fix and run `make generate`) items: type: string type: array x-kubernetes-list-type: atomic maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' format: int32 type: integer minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - - - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field and requires + the MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." format: int32 type: integer nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this value is + nil, the behavior is equivalent to the Honor policy. This + is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." type: string nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + description: "NodeTaintsPolicy indicates how we will treat + node taints when calculating pod topology spread skew. + Options are: - Honor: nodes without taints, along with + tainted nodes for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. All nodes + are included. \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is a beta-level + feature default enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." type: string topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. type: string whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered + "Unsatisfiable" for an incoming pod if and only if every + possible node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector spread + as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still be imbalanced, but + scheduler won''t make it *more* imbalanced. It''s a required + field.' type: string required: - maxSkew @@ -1605,33 +1628,32 @@ spec: type: array type: object selector: - description: |- - selector is a label query over pods that should match the replica count. - It must match the pod template's labels. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + description: 'selector is a label query over pods that should match + the replica count. It must match the pod template''s labels. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1644,10 +1666,11 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. type: object type: object x-kubernetes-map-type: atomic @@ -1677,9 +1700,8 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true storageClass: - description: |- - StorageClass defines the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + description: 'StorageClass defines the name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeClaimTemplate: description: VolumeClaimTemplate defines the volume claim template @@ -1696,9 +1718,9 @@ spec: description: EtcdStatus defines the observed state of Etcd. properties: clusterSize: - description: |- - Cluster size is the current size of the etcd cluster. - Deprecated: this field will not be populated with any value and will be removed in the future. + description: 'Cluster size is the current size of the etcd cluster. + Deprecated: this field will not be populated with any value and + will be removed in the future.' format: int32 type: integer conditions: @@ -1759,33 +1781,32 @@ spec: type: string type: object labelSelector: - description: |- - LabelSelector is a label query over pods that should match the replica count. - It must match the pod template's labels. - Deprecated: this field will be removed in the future. + description: 'LabelSelector is a label query over pods that should + match the replica count. It must match the pod template''s labels. + Deprecated: this field will be removed in the future.' properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -1798,76 +1819,17 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. type: object type: object x-kubernetes-map-type: atomic lastError: - description: |- - LastError represents the last occurred error. - Deprecated: Use LastErrors instead. + description: LastError represents the last occurred error. type: string - lastErrors: - description: LastErrors captures errors that occurred during the last - operation. - items: - description: LastError stores details of the most recent error encountered - for a resource. - properties: - code: - description: Code is an error code that uniquely identifies - an error. - type: string - description: - description: Description is a human-readable message indicating - details of the error. - type: string - observedAt: - description: ObservedAt is the time the error was observed. - format: date-time - type: string - required: - - code - - description - - observedAt - type: object - type: array - lastOperation: - description: LastOperation indicates the last operation performed - on this resource. - properties: - description: - description: Description describes the last operation. - type: string - lastUpdateTime: - description: LastUpdateTime is the time at which the operation - was updated. - format: date-time - type: string - runID: - description: |- - RunID correlates an operation with a reconciliation run. - Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is - generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this - as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering - reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. - type: string - state: - description: State is the state of the last operation. - type: string - type: - description: Type is the type of last operation. - type: string - required: - - description - - lastUpdateTime - - runID - - state - - type - type: object members: description: Members represents the members of the etcd cluster items: @@ -1925,14 +1887,13 @@ spec: format: int32 type: integer serviceName: - description: |- - ServiceName is the name of the etcd service. - Deprecated: this field will be removed in the future. + description: 'ServiceName is the name of the etcd service. Deprecated: + this field will be removed in the future.' type: string updatedReplicas: - description: |- - UpdatedReplicas is the count of updated replicas in the etcd cluster. - Deprecated: this field will be removed in the future. + description: 'UpdatedReplicas is the count of updated replicas in + the etcd cluster. Deprecated: this field will be removed in the + future.' format: int32 type: integer type: object diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 9aa0b085e..16951ef91 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -25,19 +25,14 @@ spec: source to a target store. properties: apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -46,9 +41,9 @@ spec: backups task. properties: maxBackupAge: - description: |- - MaxBackupAge is the maximum age in days that a backup must have in order to be copied. - By default all backups will be copied. + description: MaxBackupAge is the maximum age in days that a backup + must have in order to be copied. By default all backups will be + copied. format: int32 type: integer maxBackups: @@ -127,8 +122,8 @@ spec: snapshot before copying backups. type: boolean timeout: - description: |- - Timeout is the timeout for waiting for a final full snapshot. When this timeout expires, the copying of backups + description: Timeout is the timeout for waiting for a final full + snapshot. When this timeout expires, the copying of backups will be performed anyway. No timeout or 0 means wait forever. type: string required: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 140f7b07e..8bb938ba7 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.13.0 name: etcds.druid.gardener.cloud spec: group: druid.gardener.cloud @@ -14,1894 +14,1891 @@ spec: singular: etcd scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .status.ready - name: Ready - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Quorate - type: string - - jsonPath: .status.conditions[?(@.type=="AllMembersReady")].status - name: All Members Ready - type: string - - jsonPath: .status.conditions[?(@.type=="BackupReady")].status - name: Backup Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .spec.replicas - name: Cluster Size - priority: 1 - type: integer - - jsonPath: .status.currentReplicas - name: Current Replicas - priority: 1 - type: integer - - jsonPath: .status.readyReplicas - name: Ready Replicas - priority: 1 - type: integer - name: v1alpha1 - schema: - openAPIV3Schema: - description: Etcd is the Schema for the etcds API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: EtcdSpec defines the desired state of Etcd - properties: - annotations: - additionalProperties: - type: string - type: object - backup: - description: BackupSpec defines parameters associated with the full - and delta snapshots of etcd. - properties: - compactionResources: - description: |- - CompactionResources defines compute Resources required by compaction job. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - compression: - description: SnapshotCompression defines the specification for - compression of Snapshots. - properties: - enabled: - type: boolean - policy: - description: CompressionPolicy defines the type of policy - for compression of snapshots. - enum: - - gzip - - lzw - - zlib - type: string - type: object - deltaSnapshotMemoryLimit: - anyOf: - - type: integer - - type: string - description: DeltaSnapshotMemoryLimit defines the memory limit - after which delta snapshots will be taken - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - deltaSnapshotPeriod: - description: DeltaSnapshotPeriod defines the period after which - delta snapshots will be taken - type: string - deltaSnapshotRetentionPeriod: - description: |- - DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set. - The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d') - pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ - type: string - enableProfiling: - description: EnableProfiling defines if profiling should be enabled - for the etcd-backup-restore-sidecar - type: boolean - etcdSnapshotTimeout: - description: EtcdSnapshotTimeout defines the timeout duration - for etcd FullSnapshot operation - type: string - fullSnapshotSchedule: - description: FullSnapshotSchedule defines the cron standard schedule - for full snapshots. - type: string - garbageCollectionPeriod: - description: GarbageCollectionPeriod defines the period for garbage - collecting old backups - type: string - garbageCollectionPolicy: - description: GarbageCollectionPolicy defines the policy for garbage - collecting old backups - enum: - - Exponential - - LimitBased - type: string - image: - description: Image defines the etcd container image and tag - type: string - leaderElection: - description: LeaderElection defines parameters related to the - LeaderElection configuration. - properties: - etcdConnectionTimeout: - description: EtcdConnectionTimeout defines the timeout duration - for etcd client connection during leader election. - type: string - reelectionPeriod: - description: ReelectionPeriod defines the Period after which - leadership status of corresponding etcd is checked. - type: string - type: object - maxBackupsLimitBasedGC: - description: |- - MaxBackupsLimitBasedGC defines the maximum number of Full snapshots to retain in Limit Based GarbageCollectionPolicy - All full snapshots beyond this limit will be garbage collected. - format: int32 - type: integer - port: - description: Port define the port on which etcd-backup-restore - server will be exposed. - format: int32 - type: integer - resources: - description: |- - Resources defines compute Resources required by backup-restore container. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - store: - description: Store defines the specification of object store provider - for storing backups. - properties: - container: - description: Container is the name of the container the backup - is stored at. - type: string - prefix: - description: Prefix is the prefix used for the store. - type: string - provider: - description: Provider is the name of the backup provider. - type: string - secretRef: - description: SecretRef is the reference to the secret which - used to connect to the backup store. - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - prefix - type: object - tls: - description: TLSConfig hold the TLS configuration details. - properties: - clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + - additionalPrinterColumns: + - jsonPath: .status.ready + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Quorate + type: string + - jsonPath: .status.conditions[?(@.type=="AllMembersReady")].status + name: All Members Ready + type: string + - jsonPath: .status.conditions[?(@.type=="BackupReady")].status + name: Backup Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.replicas + name: Cluster Size + priority: 1 + type: integer + - jsonPath: .status.currentReplicas + name: Current Replicas + priority: 1 + type: integer + - jsonPath: .status.readyReplicas + name: Ready Replicas + priority: 1 + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: Etcd is the Schema for the etcds API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EtcdSpec defines the desired state of Etcd + properties: + annotations: + additionalProperties: + type: string + type: object + backup: + description: BackupSpec defines parameters associated with the full + and delta snapshots of etcd. + properties: + compactionResources: + description: 'CompactionResources defines compute Resources required + by compaction job. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. type: string + required: + - name type: object - x-kubernetes-map-type: atomic - tlsCASecretRef: - description: SecretReference defines a reference to a secret. + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + compression: + description: SnapshotCompression defines the specification for + compression of Snapshots. + properties: + enabled: + type: boolean + policy: + description: CompressionPolicy defines the type of policy + for compression of snapshots. + enum: + - gzip + - lzw + - zlib + type: string + type: object + deltaSnapshotMemoryLimit: + anyOf: + - type: integer + - type: string + description: DeltaSnapshotMemoryLimit defines the memory limit + after which delta snapshots will be taken + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + deltaSnapshotPeriod: + description: DeltaSnapshotPeriod defines the period after which + delta snapshots will be taken + type: string + deltaSnapshotRetentionPeriod: + description: DeltaSnapshotRetentionPeriod defines the duration + for which delta snapshots will be retained, excluding the latest + snapshot set. The value should be a string formatted as a duration + (e.g., '1s', '2m', '3h', '4d') + pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$ + type: string + enableProfiling: + description: EnableProfiling defines if profiling should be enabled + for the etcd-backup-restore-sidecar + type: boolean + etcdSnapshotTimeout: + description: EtcdSnapshotTimeout defines the timeout duration + for etcd FullSnapshot operation + type: string + fullSnapshotSchedule: + description: FullSnapshotSchedule defines the cron standard schedule + for full snapshots. + type: string + garbageCollectionPeriod: + description: GarbageCollectionPeriod defines the period for garbage + collecting old backups + type: string + garbageCollectionPolicy: + description: GarbageCollectionPolicy defines the policy for garbage + collecting old backups + enum: + - Exponential + - LimitBased + type: string + image: + description: Image defines the etcd container image and tag + type: string + leaderElection: + description: LeaderElection defines parameters related to the + LeaderElection configuration. + properties: + etcdConnectionTimeout: + description: EtcdConnectionTimeout defines the timeout duration + for etcd client connection during leader election. + type: string + reelectionPeriod: + description: ReelectionPeriod defines the Period after which + leadership status of corresponding etcd is checked. + type: string + type: object + port: + description: Port define the port on which etcd-backup-restore + server will be exposed. + format: int32 + type: integer + resources: + description: 'Resources defines compute Resources required by + backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: - dataKey: - description: DataKey is the name of the key in the data - map containing the credentials. - type: string name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. type: string + required: + - name type: object - x-kubernetes-map-type: atomic - required: - - serverTLSSecretRef - - tlsCASecretRef - type: object - type: object - etcd: - description: EtcdConfig defines parameters associated etcd deployed - properties: - authSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + store: + description: Store defines the specification of object store provider + for storing backups. + properties: + container: + description: Container is the name of the container the backup + is stored at. + type: string + prefix: + description: Prefix is the prefix used for the store. + type: string + provider: + description: Provider is the name of the backup provider. + type: string + secretRef: + description: SecretRef is the reference to the secret which + used to connect to the backup store. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - prefix + type: object + tls: + description: TLSConfig hold the TLS configuration details. + properties: + clientTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + serverTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + tlsCASecretRef: + description: SecretReference defines a reference to a secret. + properties: + dataKey: + description: DataKey is the name of the key in the data + map containing the credentials. + type: string + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - serverTLSSecretRef + - tlsCASecretRef + type: object + type: object + etcd: + description: EtcdConfig defines parameters associated etcd deployed + properties: + authSecretRef: + description: SecretReference represents a Secret Reference. It + has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + clientPort: + format: int32 + type: integer + clientService: + description: ClientService defines the parameters of the client + service that a user can specify + properties: + annotations: + additionalProperties: type: string - namespace: - description: namespace defines the space within which the - secret name must be unique. + description: Annotations specify the annotations that should + be added to the client service + type: object + labels: + additionalProperties: type: string - type: object - x-kubernetes-map-type: atomic - clientPort: - format: int32 - type: integer - clientService: - description: ClientService defines the parameters of the client - service that a user can specify - properties: - annotations: - additionalProperties: + description: Labels specify the labels that should be added + to the client service + type: object + type: object + clientUrlTls: + description: ClientUrlTLS contains the ca, server TLS and client + TLS secrets for client communication to ETCD cluster + properties: + clientTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. type: string - description: Annotations specify the annotations that should - be added to the client service - type: object - labels: - additionalProperties: + namespace: + description: namespace defines the space within which + the secret name must be unique. type: string - description: Labels specify the labels that should be added - to the client service - type: object - type: object - clientUrlTls: - description: ClientUrlTLS contains the ca, server TLS and client - TLS secrets for client communication to ETCD cluster - properties: - clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - tlsCASecretRef: - description: SecretReference defines a reference to a secret. - properties: - dataKey: - description: DataKey is the name of the key in the data - map containing the credentials. - type: string - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - serverTLSSecretRef - - tlsCASecretRef - type: object - defragmentationSchedule: - description: DefragmentationSchedule defines the cron standard - schedule for defragmentation of etcd. - type: string - etcdDefragTimeout: - description: EtcdDefragTimeout defines the timeout duration for - etcd defrag call - type: string - heartbeatDuration: - description: HeartbeatDuration defines the duration for members - to send heartbeats. The default value is 10s. - type: string - image: - description: Image defines the etcd container image and tag - type: string - metrics: - description: Metrics defines the level of detail for exported - metrics of etcd, specify 'extensive' to include histogram metrics. - enum: - - basic - - extensive - type: string - peerUrlTls: - description: |- - PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster - Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster. - properties: - clientTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - serverTLSSecretRef: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace - properties: - name: - description: name is unique within a namespace to reference - a secret resource. - type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - tlsCASecretRef: - description: SecretReference defines a reference to a secret. + type: object + x-kubernetes-map-type: atomic + serverTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + tlsCASecretRef: + description: SecretReference defines a reference to a secret. + properties: + dataKey: + description: DataKey is the name of the key in the data + map containing the credentials. + type: string + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - serverTLSSecretRef + - tlsCASecretRef + type: object + defragmentationSchedule: + description: DefragmentationSchedule defines the cron standard + schedule for defragmentation of etcd. + type: string + etcdDefragTimeout: + description: EtcdDefragTimeout defines the timeout duration for + etcd defrag call + type: string + heartbeatDuration: + description: HeartbeatDuration defines the duration for members + to send heartbeats. The default value is 10s. + type: string + image: + description: Image defines the etcd container image and tag + type: string + metrics: + description: Metrics defines the level of detail for exported + metrics of etcd, specify 'extensive' to include histogram metrics. + enum: + - basic + - extensive + type: string + peerUrlTls: + description: PeerUrlTLS contains the ca and server TLS secrets + for peer communication within ETCD cluster Currently, PeerUrlTLS + does not require client TLS secrets for gardener implementation + of ETCD cluster. + properties: + clientTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + serverTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + tlsCASecretRef: + description: SecretReference defines a reference to a secret. + properties: + dataKey: + description: DataKey is the name of the key in the data + map containing the credentials. + type: string + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which + the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - serverTLSSecretRef + - tlsCASecretRef + type: object + quota: + anyOf: + - type: integer + - type: string + description: Quota defines the etcd DB quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resources: + description: 'Resources defines the compute Resources required + by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: - dataKey: - description: DataKey is the name of the key in the data - map containing the credentials. - type: string name: - description: name is unique within a namespace to reference - a secret resource. + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. type: string - namespace: - description: namespace defines the space within which - the secret name must be unique. - type: string - type: object - x-kubernetes-map-type: atomic - required: - - serverTLSSecretRef - - tlsCASecretRef - type: object - quota: - anyOf: - - type: integer - - type: string - description: Quota defines the etcd DB quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resources: - description: |- - Resources defines the compute Resources required by etcd container. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + required: + - name type: object - type: object - serverPort: - format: int32 - type: integer - type: object - labels: - additionalProperties: - type: string - type: object - priorityClassName: - description: PriorityClassName is the name of a priority class that - shall be used for the etcd pods. + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + serverPort: + format: int32 + type: integer + type: object + labels: + additionalProperties: type: string - replicas: - format: int32 - type: integer - schedulingConstraints: - description: |- - SchedulingConstraints defines the different scheduling constraints that must be applied to the - pod spec in the etcd statefulset. - Currently supported constraints are Affinity and TopologySpreadConstraints. - properties: - affinity: - description: |- - Affinity defines the various affinity and anti-affinity rules for a pod - that are honoured by the kube-scheduler. - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. + type: object + priorityClassName: + description: PriorityClassName is the name of a priority class that + shall be used for the etcd pods. + type: string + replicas: + format: int32 + type: integer + schedulingConstraints: + description: SchedulingConstraints defines the different scheduling + constraints that must be applied to the pod spec in the etcd statefulset. + Currently supported constraints are Affinity and TopologySpreadConstraints. + properties: + affinity: + description: Affinity defines the various affinity and anti-affinity + rules for a pod that are honoured by the kube-scheduler. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. type: string - type: array - required: + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: - key - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. type: string - type: array - required: + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: - key - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object - type: array - type: object - x-kubernetes-map-type: atomic - type: array + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer required: - - nodeSelectorTerms + - podAffinityTerm + - weight type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + type: array + required: + - key + - operator type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + type: array + required: + - key + - operator type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. + type: array + matchLabels: + additionalProperties: type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node that + violates one or more of the expressions. The node that + is most preferred is the one with the greatest sum of + weights, i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + anti-affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. type: string - type: array - required: + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: - key - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. type: string - type: array - required: + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: - key - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: + required: - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the pod + will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod + label update), the system may or may not try to eventually + evict the pod from its node. When there are multiple + elements, the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + type: array + required: + - key + - operator type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + type: array + required: + - key + - operator type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. + type: array + matchLabels: + additionalProperties: type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + topologySpreadConstraints: + description: TopologySpreadConstraints describes how a group of + pods ought to spread across topology domains, that are honoured + by the kube-scheduler. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. items: type: string type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string required: - - topologyKey + - key + - operator type: object type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object type: object - type: object - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology domains, - that are honoured by the kube-scheduler. - items: - description: TopologySpreadConstraint specifies how to spread - matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - - - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - type: object - selector: - description: |- - selector is a label query over pods that should match the replica count. - It must match the pod template's labels. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. + x-kubernetes-map-type: atomic + matchLabelKeys: + description: "MatchLabelKeys is a set of pod label keys + to select the pods over which spreading will be calculated. + The keys are used to lookup values from the incoming pod + labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading + will be calculated for the incoming pod. The same key + is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't + set. Keys that don't exist in the incoming pod labels + will be ignored. A null or empty list means only match + against labelSelector. \n This is a beta field and requires + the MatchLabelKeysInPodTopologySpread feature gate to + be enabled (enabled by default)." + items: type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field and requires + the MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this value is + nil, the behavior is equivalent to the Honor policy. This + is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat + node taints when calculating pod topology spread skew. + Options are: - Honor: nodes without taints, along with + tainted nodes for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. All nodes + are included. \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is a beta-level + feature default enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered + "Unsatisfiable" for an incoming pod if and only if every + possible node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector spread + as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still be imbalanced, but + scheduler won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + selector: + description: 'selector is a label query over pods that should match + the replica count. It must match the pod template''s labels. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + type: array + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - sharedConfig: - description: SharedConfig defines parameters shared and used by Etcd - as well as backup-restore sidecar. - properties: - autoCompactionMode: - description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore - sidecar. - enum: - - periodic - - revision - type: string - autoCompactionRetention: - description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore - sidecar. + type: array + matchLabels: + additionalProperties: type: string - type: object - storageCapacity: - anyOf: - - type: integer - - type: string - description: StorageCapacity defines the size of persistent volume. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - storageClass: - description: |- - StorageClass defines the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeClaimTemplate: - description: VolumeClaimTemplate defines the volume claim template - to be created - type: string - required: - - backup - - etcd - - labels - - replicas - - selector - type: object - status: - description: EtcdStatus defines the observed state of Etcd. - properties: - clusterSize: - description: |- - Cluster size is the current size of the etcd cluster. - Deprecated: this field will not be populated with any value and will be removed in the future. - format: int32 - type: integer - conditions: - description: Conditions represents the latest available observations - of an etcd's current state. - items: - description: Condition holds the information about the state of - a resource. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateTime: - description: Last time the condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of the Etcd condition. - type: string - required: - - lastTransitionTime - - lastUpdateTime - - message - - reason - - status - - type + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. type: object - type: array - currentReplicas: - description: CurrentReplicas is the current replica count for the - etcd cluster. - format: int32 - type: integer - etcd: - description: CrossVersionObjectReference contains enough information - to let you identify the referred resource. + type: object + x-kubernetes-map-type: atomic + sharedConfig: + description: SharedConfig defines parameters shared and used by Etcd + as well as backup-restore sidecar. + properties: + autoCompactionMode: + description: AutoCompactionMode defines the auto-compaction-mode:'periodic' + mode or 'revision' mode for etcd and embedded-Etcd of backup-restore + sidecar. + enum: + - periodic + - revision + type: string + autoCompactionRetention: + description: AutoCompactionRetention defines the auto-compaction-retention + length for etcd as well as for embedded-Etcd of backup-restore + sidecar. + type: string + type: object + storageCapacity: + anyOf: + - type: integer + - type: string + description: StorageCapacity defines the size of persistent volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClass: + description: 'StorageClass defines the name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeClaimTemplate: + description: VolumeClaimTemplate defines the volume claim template + to be created + type: string + required: + - backup + - etcd + - labels + - replicas + - selector + type: object + status: + description: EtcdStatus defines the observed state of Etcd. + properties: + clusterSize: + description: 'Cluster size is the current size of the etcd cluster. + Deprecated: this field will not be populated with any value and + will be removed in the future.' + format: int32 + type: integer + conditions: + description: Conditions represents the latest available observations + of an etcd's current state. + items: + description: Condition holds the information about the state of + a resource. properties: - apiVersion: - description: API version of the referent + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time type: string - kind: - description: Kind of the referent + lastUpdateTime: + description: Last time the condition was updated. + format: date-time type: string - name: - description: Name of the referent + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of the Etcd condition. + type: string + required: + - lastTransitionTime + - lastUpdateTime + - message + - reason + - status + - type type: object - labelSelector: - description: |- - LabelSelector is a label query over pods that should match the replica count. - It must match the pod template's labels. - Deprecated: this field will be removed in the future. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + type: array + currentReplicas: + description: CurrentReplicas is the current replica count for the + etcd cluster. + format: int32 + type: integer + etcd: + description: CrossVersionObjectReference contains enough information + to let you identify the referred resource. + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: Kind of the referent + type: string + name: + description: Name of the referent + type: string + type: object + labelSelector: + description: 'LabelSelector is a label query over pods that should + match the replica count. It must match the pod template''s labels. + Deprecated: this field will be removed in the future.' + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. + type: array + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - lastError: - description: |- - LastError represents the last occurred error. - Deprecated: Use LastErrors instead. - type: string - lastErrors: - description: LastErrors captures errors that occurred during the last - operation. - items: - description: LastError stores details of the most recent error encountered - for a resource. - properties: - code: - description: Code is an error code that uniquely identifies - an error. - type: string - description: - description: Description is a human-readable message indicating - details of the error. - type: string - observedAt: - description: ObservedAt is the time the error was observed. - format: date-time - type: string - required: - - code - - description - - observedAt + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. type: object - type: array - lastOperation: - description: LastOperation indicates the last operation performed - on this resource. + type: object + x-kubernetes-map-type: atomic + lastError: + description: LastError represents the last occurred error. + type: string + members: + description: Members represents the members of the etcd cluster + items: + description: EtcdMemberStatus holds information about a etcd cluster + membership. properties: - description: - description: Description describes the last operation. + id: + description: ID is the ID of the etcd member. type: string - lastUpdateTime: - description: LastUpdateTime is the time at which the operation - was updated. + lastTransitionTime: + description: LastTransitionTime is the last time the condition's + status changed. format: date-time type: string - runID: - description: |- - RunID correlates an operation with a reconciliation run. - Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is - generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this - as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering - reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. + name: + description: Name is the name of the etcd member. It is the + name of the backing `Pod`. type: string - state: - description: State is the state of the last operation. + reason: + description: The reason for the condition's last transition. type: string - type: - description: Type is the type of last operation. + role: + description: Role is the role in the etcd cluster, either `Leader` + or `Member`. + type: string + status: + description: Status of the condition, one of True, False, Unknown. type: string required: - - description - - lastUpdateTime - - runID - - state - - type + - lastTransitionTime + - name + - reason + - status type: object - members: - description: Members represents the members of the etcd cluster - items: - description: EtcdMemberStatus holds information about a etcd cluster - membership. - properties: - id: - description: ID is the ID of the etcd member. - type: string - lastTransitionTime: - description: LastTransitionTime is the last time the condition's - status changed. - format: date-time - type: string - name: - description: Name is the name of the etcd member. It is the - name of the backing `Pod`. - type: string - reason: - description: The reason for the condition's last transition. - type: string - role: - description: Role is the role in the etcd cluster, either `Leader` - or `Member`. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - required: - - lastTransitionTime - - name - - reason - - status - type: object - type: array - observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this resource. - format: int64 - type: integer - peerUrlTLSEnabled: - description: PeerUrlTLSEnabled captures the state of peer url TLS - being enabled for the etcd member(s) - type: boolean - ready: - description: Ready is `true` if all etcd replicas are ready. - type: boolean - readyReplicas: - description: ReadyReplicas is the count of replicas being ready in - the etcd cluster. - format: int32 - type: integer - replicas: - description: Replicas is the replica count of the etcd resource. - format: int32 - type: integer - serviceName: - description: |- - ServiceName is the name of the etcd service. - Deprecated: this field will be removed in the future. - type: string - updatedReplicas: - description: |- - UpdatedReplicas is the count of updated replicas in the etcd cluster. - Deprecated: this field will be removed in the future. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - scale: - labelSelectorPath: .status.labelSelector - specReplicasPath: .spec.replicas - statusReplicasPath: .status.replicas - status: {} \ No newline at end of file + type: array + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this resource. + format: int64 + type: integer + peerUrlTLSEnabled: + description: PeerUrlTLSEnabled captures the state of peer url TLS + being enabled for the etcd member(s) + type: boolean + ready: + description: Ready is `true` if all etcd replicas are ready. + type: boolean + readyReplicas: + description: ReadyReplicas is the count of replicas being ready in + the etcd cluster. + format: int32 + type: integer + replicas: + description: Replicas is the replica count of the etcd resource. + format: int32 + type: integer + serviceName: + description: 'ServiceName is the name of the etcd service. Deprecated: + this field will be removed in the future.' + type: string + updatedReplicas: + description: 'UpdatedReplicas is the count of updated replicas in + the etcd cluster. Deprecated: this field will be removed in the + future.' + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/go.mod b/go.mod index 65433bdd5..3f2cf98db 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/onsi/gomega v1.29.0 github.com/prometheus/client_golang v1.16.0 github.com/spf13/pflag v1.0.5 + go.uber.org/mock v0.2.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 gopkg.in/yaml.v2 v2.4.0 @@ -109,7 +110,6 @@ require ( github.com/spf13/cobra v1.7.0 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 // indirect go.opencensus.io v0.24.0 // indirect - go.uber.org/mock v0.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.12.0 // indirect diff --git a/internal/common/constants.go b/internal/common/constants.go index 4b9d1b05a..f841a963f 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -19,8 +19,8 @@ const ( ) const ( - // ChartPath is the directory containing the default image vector file. - ChartPath = "charts" + // DefaultImageVectorFilePath is the path to the default image vector file. + DefaultImageVectorFilePath = "charts/images.yaml" // FinalizerName is the name of the etcd finalizer. FinalizerName = "druid.gardener.cloud/etcd-druid" ) @@ -153,7 +153,7 @@ const ( // GCSBackupVolumeMountPath is the path on a container where the GCS backup secret is mounted. GCSBackupVolumeMountPath = "/var/.gcp/" // NonGCSProviderBackupVolumeMountPath is the path on a container where the non-GCS provider backup secret is mounted. - NonGCSProviderBackupVolumeMountPath = "/var/etcd-backup/" + NonGCSProviderBackupVolumeMountPath = "/var/etcd-backup" // EtcdDataVolumeMountPath is the path on a container where the etcd data directory is hosted. EtcdDataVolumeMountPath = "/var/etcd/data" diff --git a/internal/controller/etcd/register_test.go b/internal/controller/etcd/register_test.go index 618f3e10e..fa4126f1c 100644 --- a/internal/controller/etcd/register_test.go +++ b/internal/controller/etcd/register_test.go @@ -11,8 +11,8 @@ import ( mockmanager "github.com/gardener/etcd-druid/internal/mock/controller-runtime/manager" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/golang/mock/gomock" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index c49a16bb9..7eeb623e9 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -87,7 +87,24 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Context("delete EtcdCopyBackupsTask object tests when it exists", func() { BeforeEach(func() { - task = ensureEtcdCopyBackupsTaskCreation(ctx, testTaskName, testNamespace, fakeClient) + task = testutils.CreateEtcdCopyBackupsTask(testTaskName, testNamespace, "aws", false) + + By("Create fake client with task object") + fakeClient = fakeclient.NewClientBuilder(). + WithScheme(kubernetes.Scheme). + WithObjects(task). + WithStatusSubresource(task). + Build() + + By("Ensure that copy backups task is created") + Eventually(func() error { + return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) + }).Should(Succeed()) + + r = &Reconciler{ + Client: fakeClient, + logger: logr.Discard(), + } }) AfterEach(func() { ensureEtcdCopyBackupsTaskRemoval(ctx, testTaskName, testNamespace, fakeClient) @@ -103,7 +120,11 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { It("should remove finalizer for task which does not have a corresponding job", func() { Expect(controllerutils.AddFinalizers(ctx, fakeClient, task, common.FinalizerName)).To(Succeed()) - Expect(addDeletionTimestampToTask(ctx, task, time.Now(), fakeClient)).To(Succeed()) + // use fakeClient.Delete() to simply add deletionTimestamp to `task` object, + // due to https://github.com/kubernetes-sigs/controller-runtime/pull/2316 + Expect(fakeClient.Delete(ctx, task)).To(Succeed()) + // get the updated object after deletionTimestamp has been added by fakeClient.Delete() call + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task)).To(Succeed()) _, err := r.delete(ctx, task) Expect(err).To(BeNil()) @@ -116,19 +137,15 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { job := testutils.CreateEtcdCopyBackupsJob(testTaskName, testNamespace) Expect(fakeClient.Create(ctx, job)).To(Succeed()) Expect(controllerutils.AddFinalizers(ctx, fakeClient, task, common.FinalizerName)).To(Succeed()) - Expect(addDeletionTimestampToTask(ctx, task, time.Now(), fakeClient)).To(Succeed()) + Expect(fakeClient.Delete(ctx, task)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task)).To(Succeed()) + _, err := r.delete(ctx, task) Expect(err).To(BeNil()) - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(job), job) - }).Should(BeNotFoundError()) - Eventually(func() error { - return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) - }).Should(BeNil()) + Eventually(func() error { return fakeClient.Get(ctx, client.ObjectKeyFromObject(job), job) }).Should(BeNotFoundError()) + Eventually(func() error { return fakeClient.Get(ctx, client.ObjectKeyFromObject(task), task) }).Should(BeNil()) }) - }) - }) Describe("#createJobObject", func() { diff --git a/internal/controller/manager.go b/internal/controller/manager.go index d9c77b6a2..3f2172266 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -14,6 +14,7 @@ import ( "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" "github.com/gardener/etcd-druid/internal/webhook/sentinel" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" @@ -61,15 +62,17 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { } return ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - ClientDisableCacheFor: uncachedObjects, - Scheme: kubernetes.Scheme, - MetricsBindAddress: config.Server.Metrics.BindAddress, - Host: config.Server.Webhook.BindAddress, - Port: config.Server.Webhook.Port, - CertDir: config.Server.Webhook.TLS.ServerCertDir, - LeaderElection: config.EnableLeaderElection, - LeaderElectionID: config.LeaderElectionID, - LeaderElectionResourceLock: config.LeaderElectionResourceLock, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: uncachedObjects, + }, + }, + Scheme: kubernetes.Scheme, + Metrics: metricsserver.Options{ + BindAddress: config.MetricsAddr, + }, + LeaderElection: config.EnableLeaderElection, + LeaderElectionID: config.LeaderElectionID, }) } diff --git a/internal/controller/secret/register.go b/internal/controller/secret/register.go index b15d1a92f..8b7192889 100644 --- a/internal/controller/secret/register.go +++ b/internal/controller/secret/register.go @@ -9,12 +9,12 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druidmapper "github.com/gardener/etcd-druid/internal/mapper" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/gardener/gardener/pkg/controllerutils/mapper" corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/source" ) const controllerName = "secret-controller" @@ -34,7 +34,7 @@ func (r *Reconciler) RegisterWithManager(ctx context.Context, mgr ctrl.Manager) } return c.Watch( - &source.Kind{Type: &druidv1alpha1.Etcd{}}, + source.Kind(mgr.GetCache(), &druidv1alpha1.Etcd{}), mapper.EnqueueRequestsFrom(ctx, mgr.GetCache(), druidmapper.EtcdToSecret(), mapper.UpdateWithOldAndNew, c.GetLogger()), ) } diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index bc5ca0f10..0c1af71fa 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "path/filepath" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -19,21 +18,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - // defaultImageVector is a constant for the path to the default image vector file. - defaultImageVector = "images.yaml" -) - -// getImageYAMLPath returns the path to the image vector YAML file. -// The path to the default image vector YAML path is returned, unless `useEtcdWrapperImageVector` -// is set to true, in which case the path to the etcd wrapper image vector YAML is returned. -func getImageYAMLPath() string { - return filepath.Join(common.ChartPath, defaultImageVector) -} - // CreateImageVector creates an image vector from the default images.yaml file or the images-wrapper.yaml file. func CreateImageVector() (imagevector.ImageVector, error) { - imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(getImageYAMLPath()) + imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(common.DefaultImageVectorFilePath) if err != nil { return nil, err } diff --git a/internal/health/condition/check_backup_ready_test.go b/internal/health/condition/check_backup_ready_test.go index 06a9a12f9..f0b5d297d 100644 --- a/internal/health/condition/check_backup_ready_test.go +++ b/internal/health/condition/check_backup_ready_test.go @@ -11,8 +11,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" . "github.com/gardener/etcd-druid/internal/health/condition" mockclient "github.com/gardener/etcd-druid/internal/mock/controller-runtime/client" + "go.uber.org/mock/gomock" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/health/condition/check_data_volumes_ready_test.go b/internal/health/condition/check_data_volumes_ready_test.go index b83739a00..0e4b134e5 100644 --- a/internal/health/condition/check_data_volumes_ready_test.go +++ b/internal/health/condition/check_data_volumes_ready_test.go @@ -10,8 +10,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/health/condition" mockclient "github.com/gardener/etcd-druid/internal/mock/controller-runtime/client" + "go.uber.org/mock/gomock" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" diff --git a/internal/health/etcdmember/check_ready_test.go b/internal/health/etcdmember/check_ready_test.go index 905ef89b5..15865e953 100644 --- a/internal/health/etcdmember/check_ready_test.go +++ b/internal/health/etcdmember/check_ready_test.go @@ -17,10 +17,10 @@ import ( kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/test" "github.com/go-logr/logr" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gstruct" + "go.uber.org/mock/gomock" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/internal/mock/controller-runtime/client/doc.go b/internal/mock/controller-runtime/client/doc.go index 44a22b400..21ee8c39e 100644 --- a/internal/mock/controller-runtime/client/doc.go +++ b/internal/mock/controller-runtime/client/doc.go @@ -2,4 +2,5 @@ // // SPDX-License-Identifier: Apache-2.0 +//go:generate mockgen -package client -destination=mocks.go sigs.k8s.io/controller-runtime/pkg/client Client,StatusWriter,Reader,Writer package client diff --git a/internal/mock/controller-runtime/client/mocks.go b/internal/mock/controller-runtime/client/mocks.go index a43241564..a4077a7bb 100644 --- a/internal/mock/controller-runtime/client/mocks.go +++ b/internal/mock/controller-runtime/client/mocks.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" meta "k8s.io/apimachinery/pkg/api/meta" runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" types "k8s.io/apimachinery/pkg/types" client "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -114,6 +115,36 @@ func (mr *MockClientMockRecorder) Get(arg0, arg1, arg2 interface{}, arg3 ...inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), varargs...) } +// GroupVersionKindFor mocks base method. +func (m *MockClient) GroupVersionKindFor(arg0 runtime.Object) (schema.GroupVersionKind, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GroupVersionKindFor", arg0) + ret0, _ := ret[0].(schema.GroupVersionKind) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GroupVersionKindFor indicates an expected call of GroupVersionKindFor. +func (mr *MockClientMockRecorder) GroupVersionKindFor(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GroupVersionKindFor", reflect.TypeOf((*MockClient)(nil).GroupVersionKindFor), arg0) +} + +// IsObjectNamespaced mocks base method. +func (m *MockClient) IsObjectNamespaced(arg0 runtime.Object) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsObjectNamespaced", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsObjectNamespaced indicates an expected call of IsObjectNamespaced. +func (mr *MockClientMockRecorder) IsObjectNamespaced(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsObjectNamespaced", reflect.TypeOf((*MockClient)(nil).IsObjectNamespaced), arg0) +} + // List mocks base method. func (m *MockClient) List(arg0 context.Context, arg1 client.ObjectList, arg2 ...client.ListOption) error { m.ctrl.T.Helper() diff --git a/internal/mock/controller-runtime/manager/mocks.go b/internal/mock/controller-runtime/manager/mocks.go index dc8dd8d88..2566483a4 100644 --- a/internal/mock/controller-runtime/manager/mocks.go +++ b/internal/mock/controller-runtime/manager/mocks.go @@ -10,14 +10,14 @@ import ( reflect "reflect" logr "github.com/go-logr/logr" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" meta "k8s.io/apimachinery/pkg/api/meta" runtime "k8s.io/apimachinery/pkg/runtime" rest "k8s.io/client-go/rest" record "k8s.io/client-go/tools/record" cache "sigs.k8s.io/controller-runtime/pkg/cache" client "sigs.k8s.io/controller-runtime/pkg/client" - v1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + config "sigs.k8s.io/controller-runtime/pkg/config" healthz "sigs.k8s.io/controller-runtime/pkg/healthz" manager "sigs.k8s.io/controller-runtime/pkg/manager" webhook "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -74,20 +74,6 @@ func (mr *MockManagerMockRecorder) AddHealthzCheck(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddHealthzCheck", reflect.TypeOf((*MockManager)(nil).AddHealthzCheck), arg0, arg1) } -// AddMetricsExtraHandler mocks base method. -func (m *MockManager) AddMetricsExtraHandler(arg0 string, arg1 http.Handler) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddMetricsExtraHandler", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// AddMetricsExtraHandler indicates an expected call of AddMetricsExtraHandler. -func (mr *MockManagerMockRecorder) AddMetricsExtraHandler(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddMetricsExtraHandler", reflect.TypeOf((*MockManager)(nil).AddMetricsExtraHandler), arg0, arg1) -} - // AddReadyzCheck mocks base method. func (m *MockManager) AddReadyzCheck(arg0 string, arg1 healthz.Checker) error { m.ctrl.T.Helper() @@ -173,10 +159,10 @@ func (mr *MockManagerMockRecorder) GetConfig() *gomock.Call { } // GetControllerOptions mocks base method. -func (m *MockManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { +func (m *MockManager) GetControllerOptions() config.Controller { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetControllerOptions") - ret0, _ := ret[0].(v1alpha1.ControllerConfigurationSpec) + ret0, _ := ret[0].(config.Controller) return ret0 } @@ -214,6 +200,20 @@ func (mr *MockManagerMockRecorder) GetFieldIndexer() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFieldIndexer", reflect.TypeOf((*MockManager)(nil).GetFieldIndexer)) } +// GetHTTPClient mocks base method. +func (m *MockManager) GetHTTPClient() *http.Client { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHTTPClient") + ret0, _ := ret[0].(*http.Client) + return ret0 +} + +// GetHTTPClient indicates an expected call of GetHTTPClient. +func (mr *MockManagerMockRecorder) GetHTTPClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHTTPClient", reflect.TypeOf((*MockManager)(nil).GetHTTPClient)) +} + // GetLogger mocks base method. func (m *MockManager) GetLogger() logr.Logger { m.ctrl.T.Helper() @@ -257,10 +257,10 @@ func (mr *MockManagerMockRecorder) GetScheme() *gomock.Call { } // GetWebhookServer mocks base method. -func (m *MockManager) GetWebhookServer() *webhook.Server { +func (m *MockManager) GetWebhookServer() webhook.Server { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetWebhookServer") - ret0, _ := ret[0].(*webhook.Server) + ret0, _ := ret[0].(webhook.Server) return ret0 } @@ -270,20 +270,6 @@ func (mr *MockManagerMockRecorder) GetWebhookServer() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWebhookServer", reflect.TypeOf((*MockManager)(nil).GetWebhookServer)) } -// SetFields mocks base method. -func (m *MockManager) SetFields(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetFields", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetFields indicates an expected call of SetFields. -func (mr *MockManagerMockRecorder) SetFields(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFields", reflect.TypeOf((*MockManager)(nil).SetFields), arg0) -} - // Start mocks base method. func (m *MockManager) Start(arg0 context.Context) error { m.ctrl.T.Helper() diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index 2a0caeb9e..d1b0e46e5 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -6,7 +6,6 @@ package utils import ( "fmt" - "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" @@ -60,23 +59,22 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) return nil, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") } - credentialsMountPath := strings.TrimSuffix(common.NonGCSProviderBackupVolumeMountPath, "/") switch provider { case S3: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, credentialsMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) case ABS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, credentialsMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) case GCS: envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.GCSBackupVolumeMountPath))) envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) case Swift: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, credentialsMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) case OSS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, credentialsMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) case ECS: if store.SecretRef == nil { @@ -87,7 +85,7 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) case OCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, credentialsMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) } return envVars, nil diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index e49398463..14d4769b6 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -41,11 +41,7 @@ type Handler struct { // NewHandler creates a new handler for Sentinel Webhook. func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { - decoder, err := admission.NewDecoder(mgr.GetScheme()) - if err != nil { - return nil, err - } - + decoder := admission.NewDecoder(mgr.GetScheme()) return &Handler{ Client: mgr.GetClient(), config: config, diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index 9bfa4ccb7..c2b5f49ab 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -71,8 +71,7 @@ func TestHandleCreateAndConnect(t *testing.T) { } cl := testutils.CreateDefaultFakeClient() - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -93,7 +92,7 @@ func TestHandleCreateAndConnect(t *testing.T) { }, }) g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(string(resp.Result.Reason)).To(Equal(tc.expectedMsg)) + g.Expect(resp.Result.Message).To(Equal(tc.expectedMsg)) }) } } @@ -101,8 +100,7 @@ func TestHandleCreateAndConnect(t *testing.T) { func TestHandleLeaseUpdate(t *testing.T) { g := NewWithT(t) cl := fake.NewClientBuilder().Build() - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -121,15 +119,14 @@ func TestHandleLeaseUpdate(t *testing.T) { }) g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(string(resp.Result.Reason)).To(Equal("lease resource can be freely updated")) + g.Expect(resp.Result.Message).To(Equal("lease resource can be freely updated")) } func TestUnexpectedResourceType(t *testing.T) { g := NewWithT(t) cl := fake.NewClientBuilder().Build() - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -148,15 +145,14 @@ func TestUnexpectedResourceType(t *testing.T) { }) g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(string(resp.Result.Reason)).To(Equal("unexpected resource type: coordination.k8s.io/Unknown")) + g.Expect(resp.Result.Message).To(Equal("unexpected resource type: coordination.k8s.io/Unknown")) } func TestMissingResourcePartOfLabel(t *testing.T) { g := NewWithT(t) cl := fake.NewClientBuilder().Build() - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -181,7 +177,7 @@ func TestMissingResourcePartOfLabel(t *testing.T) { }) g.Expect(response.Allowed).To(Equal(true)) - g.Expect(string(response.Result.Reason)).To(Equal(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey))) + g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey))) } func TestHandleUpdate(t *testing.T) { @@ -202,7 +198,6 @@ func TestHandleUpdate(t *testing.T) { exemptServiceAccounts []string // ----- expected ----- expectedAllowed bool - expectedReason string expectedMessage string expectedCode int32 }{ @@ -211,7 +206,7 @@ func TestHandleUpdate(t *testing.T) { objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, expectedAllowed: true, - expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { @@ -221,7 +216,7 @@ func TestHandleUpdate(t *testing.T) { etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: false, - expectedReason: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, { @@ -231,7 +226,7 @@ func TestHandleUpdate(t *testing.T) { etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: true, - expectedReason: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), + expectedMessage: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), expectedCode: http.StatusOK, }, { @@ -241,7 +236,7 @@ func TestHandleUpdate(t *testing.T) { reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, - expectedReason: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { @@ -251,7 +246,7 @@ func TestHandleUpdate(t *testing.T) { reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: false, - expectedReason: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, } @@ -265,8 +260,7 @@ func TestHandleUpdate(t *testing.T) { Build() cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -293,7 +287,6 @@ func TestHandleUpdate(t *testing.T) { }) g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) - g.Expect(string(response.Result.Reason)).To(Equal(tc.expectedReason)) g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) g.Expect(response.Result.Code).To(Equal(tc.expectedCode)) }) @@ -348,8 +341,7 @@ func TestHandleWithInvalidRequestObject(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cl := testutils.CreateDefaultFakeClient() - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -413,8 +405,7 @@ func TestEtcdGetFailures(t *testing.T) { for _, tc := range testCases { t.Run(t.Name(), func(t *testing.T) { cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, @@ -441,7 +432,6 @@ func TestEtcdGetFailures(t *testing.T) { }) g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) - g.Expect(string(response.Result.Reason)).To(Equal(tc.expectedReason)) g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) g.Expect(response.Result.Code).To(Equal(tc.expectedCode)) }) @@ -475,7 +465,7 @@ func TestHandleDelete(t *testing.T) { objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, expectedAllowed: true, - expectedReason: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { @@ -485,7 +475,8 @@ func TestHandleDelete(t *testing.T) { etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: false, - expectedReason: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), + expectedReason: "Forbidden", + expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, { @@ -495,7 +486,7 @@ func TestHandleDelete(t *testing.T) { etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: true, - expectedReason: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), + expectedMessage: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), expectedCode: http.StatusOK, }, { @@ -505,7 +496,7 @@ func TestHandleDelete(t *testing.T) { reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, - expectedReason: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { @@ -515,7 +506,8 @@ func TestHandleDelete(t *testing.T) { reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: false, - expectedReason: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedReason: "Forbidden", expectedCode: http.StatusForbidden, }, } @@ -528,8 +520,7 @@ func TestHandleDelete(t *testing.T) { Build() cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder, err := admission.NewDecoder(cl.Scheme()) - g.Expect(err).ToNot(HaveOccurred()) + decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ Client: cl, diff --git a/test/it/controller/assets/assets.go b/test/it/controller/assets/assets.go index bd62035ce..34f5179d2 100644 --- a/test/it/controller/assets/assets.go +++ b/test/it/controller/assets/assets.go @@ -35,7 +35,7 @@ func GetEtcdChartPath() string { // CreateImageVector creates an image vector. func CreateImageVector(g *WithT) imagevector.ImageVector { - imageVectorPath := filepath.Join("..", "..", "..", "..", common.ChartPath, "images.yaml") + imageVectorPath := filepath.Join("..", "..", "..", "..", common.DefaultImageVectorFilePath) imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(imageVectorPath) g.Expect(err).To(BeNil()) return imageVector diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index b9a1a47a4..337067f13 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -10,18 +10,18 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" testutils "github.com/gardener/etcd-druid/test/utils" + eventsv1 "k8s.io/api/events/v1" + eventsv1beta1 "k8s.io/api/events/v1beta1" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/go-logr/logr" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - eventsv1 "k8s.io/api/events/v1" - eventsv1beta1 "k8s.io/api/events/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -124,24 +124,22 @@ func (t *itTestEnv) createTestEnvironment(scheme *k8sruntime.Scheme, crdDirector func (t *itTestEnv) createManager(scheme *k8sruntime.Scheme, clientBuilder *testutils.TestClientBuilder) { mgr, err := manager.New(t.config, manager.Options{ - Scheme: scheme, - MetricsBindAddress: "0", - ClientDisableCacheFor: []client.Object{ - &corev1.Event{}, - &eventsv1beta1.Event{}, - &eventsv1.Event{}, + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", }, - NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { + options.Cache.DisableFor = []client.Object{ + &corev1.Event{}, + &eventsv1beta1.Event{}, + &eventsv1.Event{}, + } cl, err := client.New(config, options) if err != nil { return nil, err } testCl := clientBuilder.WithClient(cl).Build() - return client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cache, - Client: testCl, - UncachedObjects: uncachedObjects, - }) + return testCl, nil }, }) t.g.Expect(err).ToNot(HaveOccurred()) diff --git a/test/utils/client.go b/test/utils/client.go index cabef09d1..6a1682c3e 100644 --- a/test/utils/client.go +++ b/test/utils/client.go @@ -285,6 +285,14 @@ func (c *testClient) RESTMapper() meta.RESTMapper { return c.delegate.RESTMapper() } +func (c *testClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.delegate.GroupVersionKindFor(obj) +} + +func (c *testClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.delegate.IsObjectNamespaced(obj) +} + // ---------------------------------- Helper methods ---------------------------------- func (c *testClient) getRecordedObjectError(method ClientMethod, objKey client.ObjectKey) error { for _, errRecord := range c.errorRecords { diff --git a/vendor/cloud.google.com/go/.gitignore b/vendor/cloud.google.com/go/.gitignore deleted file mode 100644 index cc7e53b46..000000000 --- a/vendor/cloud.google.com/go/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -# Editors -.idea -.vscode -*.swp -.history - -# Test files -*.test -coverage.txt - -# Other -.DS_Store diff --git a/vendor/cloud.google.com/go/.release-please-manifest-submodules.json b/vendor/cloud.google.com/go/.release-please-manifest-submodules.json deleted file mode 100644 index 355c70d1a..000000000 --- a/vendor/cloud.google.com/go/.release-please-manifest-submodules.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "accessapproval": "1.4.0", - "accesscontextmanager": "1.3.0", - "aiplatform": "1.24.0", - "analytics": "0.12.0", - "apigateway": "1.3.0", - "apigeeconnect": "1.3.0", - "apigeeregistry": "0.2.1", - "apikeys": "0.2.0", - "appengine": "1.4.0", - "area120": "0.6.0", - "artifactregistry": "1.8.0", - "asset": "1.9.0", - "assuredworkloads": "1.8.0", - "automl": "1.7.0", - "baremetalsolution": "0.3.0", - "batch": "0.3.0", - "beyondcorp": "0.2.0", - "billing": "1.6.0", - "binaryauthorization": "1.3.0", - "certificatemanager": "1.3.0", - "channel": "1.8.0", - "cloudbuild": "1.3.0", - "clouddms": "1.3.0", - "cloudtasks": "1.7.0", - "compute": "1.12.1", - "compute/metadata": "0.1.1", - "contactcenterinsights": "1.3.0", - "container": "1.6.0", - "containeranalysis": "0.6.0", - "datacatalog": "1.7.0", - "dataflow": "0.7.0", - "dataform": "0.5.0", - "datafusion": "1.4.0", - "datalabeling": "0.6.0", - "dataplex": "1.3.0", - "dataproc": "1.7.0", - "dataqna": "0.6.0", - "datastream": "1.4.0", - "deploy": "1.4.0", - "dialogflow": "1.18.0", - "dlp": "1.6.0", - "documentai": "1.9.0", - "domains": "0.7.0", - "edgecontainer": "0.2.0", - "essentialcontacts": "1.3.0", - "eventarc": "1.7.0", - "filestore": "1.3.0", - "functions": "1.8.0", - "gaming": "1.7.0", - "gkebackup": "0.2.0", - "gkeconnect": "0.6.0", - "gkehub": "0.10.0", - "gkemulticloud": "0.3.0", - "grafeas": "0.2.0", - "gsuiteaddons": "1.3.0", - "iam": "0.6.0", - "iap": "1.4.0", - "ids": "1.1.0", - "iot": "1.3.0", - "kms": "1.5.0", - "language": "1.7.0", - "lifesciences": "0.6.0", - "managedidentities": "1.3.0", - "mediatranslation": "0.6.0", - "memcache": "1.6.0", - "metastore": "1.7.0", - "monitoring": "1.7.0", - "networkconnectivity": "1.6.0", - "networkmanagement": "1.4.0", - "networksecurity": "0.6.0", - "notebooks": "1.4.0", - "optimization": "1.1.0", - "orchestration": "1.3.0", - "orgpolicy": "1.4.0", - "osconfig": "1.9.0", - "oslogin": "1.6.0", - "phishingprotection": "0.6.0", - "policytroubleshooter": "1.3.0", - "privatecatalog": "0.6.0", - "recaptchaenterprise/v2": "2.4.0", - "recommendationengine": "0.6.0", - "recommender": "1.7.0", - "redis": "1.9.0", - "resourcemanager": "1.3.0", - "resourcesettings": "1.3.0", - "retail": "1.10.0", - "run": "0.2.0", - "scheduler": "1.6.0", - "secretmanager": "1.8.0", - "security": "1.9.0", - "securitycenter": "1.15.0", - "servicecontrol": "1.4.0", - "servicedirectory": "1.6.0", - "servicemanagement": "1.4.0", - "serviceusage": "1.3.0", - "shell": "1.3.0", - "speech": "1.8.0", - "storagetransfer": "1.5.0", - "talent": "1.3.0", - "texttospeech": "1.4.0", - "tpu": "1.3.0", - "trace": "1.3.0", - "translate": "1.3.0", - "video": "1.8.0", - "videointelligence": "1.8.0", - "vision/v2": "2.4.0", - "vmmigration": "1.2.0", - "vpcaccess": "1.4.0", - "webrisk": "1.6.0", - "websecurityscanner": "1.3.0", - "workflows": "1.8.0" -} diff --git a/vendor/cloud.google.com/go/.release-please-manifest.json b/vendor/cloud.google.com/go/.release-please-manifest.json deleted file mode 100644 index d9c938906..000000000 --- a/vendor/cloud.google.com/go/.release-please-manifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - ".": "0.104.0" -} diff --git a/vendor/cloud.google.com/go/CHANGES.md b/vendor/cloud.google.com/go/CHANGES.md deleted file mode 100644 index 8d00d2e95..000000000 --- a/vendor/cloud.google.com/go/CHANGES.md +++ /dev/null @@ -1,2424 +0,0 @@ -# Changes - -## [0.104.0](https://github.com/googleapis/google-cloud-go/compare/v0.103.0...v0.104.0) (2022-08-24) - - -### Features - -* **godocfx:** add friendlyAPIName ([#6447](https://github.com/googleapis/google-cloud-go/issues/6447)) ([c6d3ba4](https://github.com/googleapis/google-cloud-go/commit/c6d3ba401b7b3ae9b710a8850c6ec5d49c4c1490)) - -## [0.103.0](https://github.com/googleapis/google-cloud-go/compare/v0.102.1...v0.103.0) (2022-06-29) - - -### Features - -* **privateca:** temporarily remove REGAPIC support ([199b725](https://github.com/googleapis/google-cloud-go/commit/199b7250f474b1a6f53dcf0aac0c2966f4987b68)) - -## [0.102.1](https://github.com/googleapis/google-cloud-go/compare/v0.102.0...v0.102.1) (2022-06-17) - - -### Bug Fixes - -* **longrunning:** regapic remove path params duped as query params ([#6183](https://github.com/googleapis/google-cloud-go/issues/6183)) ([c963be3](https://github.com/googleapis/google-cloud-go/commit/c963be301f074779e6bb8c897d8064fa076e9e35)) - -## [0.102.0](https://github.com/googleapis/google-cloud-go/compare/v0.101.1...v0.102.0) (2022-05-24) - - -### Features - -* **civil:** add Before and After methods to civil.Time ([#5703](https://github.com/googleapis/google-cloud-go/issues/5703)) ([7acaaaf](https://github.com/googleapis/google-cloud-go/commit/7acaaafef47668c3e8382b8bc03475598c3db187)) - -### [0.101.1](https://github.com/googleapis/google-cloud-go/compare/v0.101.0...v0.101.1) (2022-05-03) - - -### Bug Fixes - -* **internal/gapicgen:** properly update modules that have no gapic changes ([#5945](https://github.com/googleapis/google-cloud-go/issues/5945)) ([de2befc](https://github.com/googleapis/google-cloud-go/commit/de2befcaa2a886499db9da6d4d04d28398c8d44b)) - -## [0.101.0](https://github.com/googleapis/google-cloud-go/compare/v0.100.2...v0.101.0) (2022-04-20) - - -### Features - -* **all:** bump grpc dep ([#5481](https://github.com/googleapis/google-cloud-go/issues/5481)) ([b12964d](https://github.com/googleapis/google-cloud-go/commit/b12964df5c63c647aaf204e73cfcdfd379d19682)) -* **internal/gapicgen:** change versionClient for gapics ([#5687](https://github.com/googleapis/google-cloud-go/issues/5687)) ([55f0d92](https://github.com/googleapis/google-cloud-go/commit/55f0d92bf112f14b024b4ab0076c9875a17423c9)) - - -### Bug Fixes - -* **internal/gapicgen:** add generation of internal/version.go for new client modules ([#5726](https://github.com/googleapis/google-cloud-go/issues/5726)) ([341e0df](https://github.com/googleapis/google-cloud-go/commit/341e0df1e44480706180cc5b07c49b3cee904095)) -* **internal/gapicgen:** don't gen version files for longrunning and debugger ([#5698](https://github.com/googleapis/google-cloud-go/issues/5698)) ([3a81108](https://github.com/googleapis/google-cloud-go/commit/3a81108c74cd8864c56b8ab5939afd864db3c64b)) -* **internal/gapicgen:** don't try to make snippets for non-gapics ([#5919](https://github.com/googleapis/google-cloud-go/issues/5919)) ([c94dddc](https://github.com/googleapis/google-cloud-go/commit/c94dddc60ef83a0584ba8f7dd24589d9db971672)) -* **internal/gapicgen:** move breaking change indicator if present ([#5452](https://github.com/googleapis/google-cloud-go/issues/5452)) ([e712df5](https://github.com/googleapis/google-cloud-go/commit/e712df5ebb45598a1653081d7e11e578bad22ff8)) -* **internal/godocfx:** prevent errors for filtered mods ([#5485](https://github.com/googleapis/google-cloud-go/issues/5485)) ([6cb9b89](https://github.com/googleapis/google-cloud-go/commit/6cb9b89b2d654c695eab00d8fb375cce0cd6e059)) - -## [0.100.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.99.0...v0.100.0) (2022-01-04) - - -### Features - -* **analytics/admin:** add the `AcknowledgeUserDataCollection` operation which acknowledges the terms of user data collection for the specified property feat: add the new resource type `DataStream`, which is planned to eventually replace `WebDataStream`, `IosAppDataStream`, `AndroidAppDataStream` resources fix!: remove `GetEnhancedMeasurementSettings`, `UpdateEnhancedMeasurementSettingsRequest`, `UpdateEnhancedMeasurementSettingsRequest` operations from the API feat: add `CreateDataStream`, `DeleteDataStream`, `UpdateDataStream`, `ListDataStreams` operations to support the new `DataStream` resource feat: add `DISPLAY_VIDEO_360_ADVERTISER_LINK`, `DISPLAY_VIDEO_360_ADVERTISER_LINK_PROPOSAL` fields to `ChangeHistoryResourceType` enum feat: add the `account` field to the `Property` type docs: update the documentation with a new list of valid values for `UserLink.direct_roles` field ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) -* **assuredworkloads:** EU Regions and Support With Sovereign Controls ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) -* **dialogflow/cx:** added the display name of the current page in webhook requests ([e0833b2](https://www.github.com/googleapis/google-cloud-go/commit/e0833b2853834ba79fd20ca2ae9c613d585dd2a5)) -* **dialogflow/cx:** added the display name of the current page in webhook requests ([e0833b2](https://www.github.com/googleapis/google-cloud-go/commit/e0833b2853834ba79fd20ca2ae9c613d585dd2a5)) -* **dialogflow:** added export documentation method feat: added filter in list documentations request feat: added option to import custom metadata from Google Cloud Storage in reload document request feat: added option to apply partial update to the smart messaging allowlist in reload document request feat: added filter in list knowledge bases request ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) -* **dialogflow:** removed OPTIONAL for speech model variant docs: added more docs for speech model variant and improved docs format for participant ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) -* **recaptchaenterprise:** add new reCAPTCHA Enterprise fraud annotations ([3dd34a2](https://www.github.com/googleapis/google-cloud-go/commit/3dd34a262edbff63b9aece8faddc2ff0d98ce42a)) - - -### Bug Fixes - -* **artifactregistry:** fix resource pattern ID segment name ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) -* **compute:** add parameter in compute bazel rules ([#692](https://www.github.com/googleapis/google-cloud-go/issues/692)) ([5444809](https://www.github.com/googleapis/google-cloud-go/commit/5444809e0b7cf9f5416645ea2df6fec96f8b9023)) -* **profiler:** refine regular expression for parsing backoff duration in E2E tests ([#5229](https://www.github.com/googleapis/google-cloud-go/issues/5229)) ([4438aeb](https://www.github.com/googleapis/google-cloud-go/commit/4438aebca2ec01d4dbf22287aa651937a381e043)) -* **profiler:** remove certificate expiration workaround ([#5222](https://www.github.com/googleapis/google-cloud-go/issues/5222)) ([2da36c9](https://www.github.com/googleapis/google-cloud-go/commit/2da36c95f44d5f88fd93cd949ab78823cea74fe7)) - -## [0.99.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.98.0...v0.99.0) (2021-12-06) - - -### Features - -* **dialogflow/cx:** added `TelephonyTransferCall` in response message ([fe27098](https://www.github.com/googleapis/google-cloud-go/commit/fe27098e5d429911428821ded57384353e699774)) - -## [0.98.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.97.0...v0.98.0) (2021-12-03) - - -### Features - -* **aiplatform:** add enable_private_service_connect field to Endpoint feat: add id field to DeployedModel feat: add service_attachment field to PrivateEndpoints feat: add endpoint_id to CreateEndpointRequest and method signature to CreateEndpoint feat: add method signature to CreateFeatureStore, CreateEntityType, CreateFeature feat: add network and enable_private_service_connect to IndexEndpoint feat: add service_attachment to IndexPrivateEndpoints feat: add stratified_split field to training_pipeline InputDataConfig ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) -* **aiplatform:** add featurestore service to aiplatform v1 feat: add metadata service to aiplatform v1 ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) -* **aiplatform:** Adds support for `google.protobuf.Value` pipeline parameters in the `parameter_values` field ([88a1cdb](https://www.github.com/googleapis/google-cloud-go/commit/88a1cdbef3cc337354a61bc9276725bfb9a686d8)) -* **aiplatform:** Tensorboard v1 protos release feat:Exposing a field for v1 CustomJob-Tensorboard integration. ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) -* **binaryauthorization:** add new admission rule types to Policy feat: update SignatureAlgorithm enum to match algorithm names in KMS feat: add SystemPolicyV1Beta1 service ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **channel:** add resource type to ChannelPartnerLink ([c206948](https://www.github.com/googleapis/google-cloud-go/commit/c2069487f6af5bcb37d519afeb60e312e35e67d5)) -* **cloudtasks:** add C++ rules for Cloud Tasks ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) -* **compute:** Move compute.v1 from googleapis-discovery to googleapis ([#675](https://www.github.com/googleapis/google-cloud-go/issues/675)) ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **compute:** Switch to string enums for compute ([#685](https://www.github.com/googleapis/google-cloud-go/issues/685)) ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) -* **contactcenterinsights:** Add ability to update phrase matchers feat: Add issue model stats to time series feat: Add display name to issue model stats ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **contactcenterinsights:** Add WriteDisposition to BigQuery Export API ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) -* **contactcenterinsights:** deprecate issue_matches docs: if conversation medium is unspecified, it will default to PHONE_CALL ([1a0720f](https://www.github.com/googleapis/google-cloud-go/commit/1a0720f2f33bb14617f5c6a524946a93209e1266)) -* **contactcenterinsights:** new feature flag disable_issue_modeling docs: fixed formatting issues in the reference documentation ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) -* **contactcenterinsights:** remove feature flag disable_issue_modeling ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) -* **datacatalog:** Added BigQueryDateShardedSpec.latest_shard_resource field feat: Added SearchCatalogResult.display_name field feat: Added SearchCatalogResult.description field ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **dataproc:** add Dataproc Serverless for Spark Batches API ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) -* **dataproc:** Add support for dataproc BatchController service ([8519b94](https://www.github.com/googleapis/google-cloud-go/commit/8519b948fee5dc82d39300c4d96e92c85fe78fe6)) -* **dialogflow/cx:** added API for changelogs docs: clarified semantic of the streaming APIs ([587bba5](https://www.github.com/googleapis/google-cloud-go/commit/587bba5ad792a92f252107aa38c6af50fb09fb58)) -* **dialogflow/cx:** added API for changelogs docs: clarified semantic of the streaming APIs ([587bba5](https://www.github.com/googleapis/google-cloud-go/commit/587bba5ad792a92f252107aa38c6af50fb09fb58)) -* **dialogflow/cx:** added support for comparing between versions docs: clarified security settings API reference ([83b941c](https://www.github.com/googleapis/google-cloud-go/commit/83b941c0983e44fdd18ceee8c6f3e91219d72ad1)) -* **dialogflow/cx:** added support for Deployments with ListDeployments and GetDeployment apis feat: added support for DeployFlow api under Environments feat: added support for TestCasesConfig under Environment docs: added long running operation explanation for several apis fix!: marked resource name of security setting as not-required ([8c5c6cf](https://www.github.com/googleapis/google-cloud-go/commit/8c5c6cf9df046b67998a8608d05595bd9e34feb0)) -* **dialogflow/cx:** allow setting custom CA for generic webhooks and release CompareVersions API docs: clarify DLP template reader usage ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) -* **dialogflow:** added support to configure security settings, language code and time zone on conversation profile ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **dialogflow:** support document metadata filter in article suggestion and smart reply model in human agent assistant ([e33350c](https://www.github.com/googleapis/google-cloud-go/commit/e33350cfcabcddcda1a90069383d39c68deb977a)) -* **dlp:** added deidentify replacement dictionaries feat: added field for BigQuery inspect template inclusion lists feat: added field to support infotype versioning ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) -* **domains:** added library for Cloud Domains v1 API. Also added methods for the transfer-in flow docs: improved API comments ([8519b94](https://www.github.com/googleapis/google-cloud-go/commit/8519b948fee5dc82d39300c4d96e92c85fe78fe6)) -* **functions:** Secret Manager integration fields 'secret_environment_variables' and 'secret_volumes' added feat: CMEK integration fields 'kms_key_name' and 'docker_repository' added ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **kms:** add OAEP+SHA1 to the list of supported algorithms ([8c5c6cf](https://www.github.com/googleapis/google-cloud-go/commit/8c5c6cf9df046b67998a8608d05595bd9e34feb0)) -* **kms:** add RPC retry information for MacSign, MacVerify, and GenerateRandomBytes Committer: [@bdhess](https://www.github.com/bdhess) ([1a0720f](https://www.github.com/googleapis/google-cloud-go/commit/1a0720f2f33bb14617f5c6a524946a93209e1266)) -* **kms:** add support for Raw PKCS[#1](https://www.github.com/googleapis/google-cloud-go/issues/1) signing keys ([58bea89](https://www.github.com/googleapis/google-cloud-go/commit/58bea89a3d177d5c431ff19310794e3296253353)) -* **monitoring/apiv3:** add CreateServiceTimeSeries RPC ([9e41088](https://www.github.com/googleapis/google-cloud-go/commit/9e41088bb395fbae0e757738277d5c95fa2749c8)) -* **monitoring/dashboard:** Added support for auto-close configurations ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) -* **monitoring/metricsscope:** promote apiv1 to GA ([#5135](https://www.github.com/googleapis/google-cloud-go/issues/5135)) ([33c0f63](https://www.github.com/googleapis/google-cloud-go/commit/33c0f63e0e0ce69d9ef6e57b04d1b8cc10ed2b78)) -* **osconfig:** OSConfig: add OS policy assignment rpcs ([83b941c](https://www.github.com/googleapis/google-cloud-go/commit/83b941c0983e44fdd18ceee8c6f3e91219d72ad1)) -* **osconfig:** Update OSConfig API ([e33350c](https://www.github.com/googleapis/google-cloud-go/commit/e33350cfcabcddcda1a90069383d39c68deb977a)) -* **osconfig:** Update osconfig v1 and v1alpha RecurringSchedule.Frequency with DAILY frequency ([59e548a](https://www.github.com/googleapis/google-cloud-go/commit/59e548acc249c7bddd9c884c2af35d582a408c4d)) -* **recaptchaenterprise:** add reCAPTCHA Enterprise account defender API methods ([88a1cdb](https://www.github.com/googleapis/google-cloud-go/commit/88a1cdbef3cc337354a61bc9276725bfb9a686d8)) -* **redis:** [Cloud Memorystore for Redis] Support Multiple Read Replicas when creating Instance ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **redis:** [Cloud Memorystore for Redis] Support Multiple Read Replicas when creating Instance ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **security/privateca:** add IAMPolicy & Locations mix-in support ([1a0720f](https://www.github.com/googleapis/google-cloud-go/commit/1a0720f2f33bb14617f5c6a524946a93209e1266)) -* **securitycenter:** Added a new API method UpdateExternalSystem, which enables updating a finding w/ external system metadata. External systems are a child resource under finding, and are housed on the finding itself, and can also be filtered on in Notifications, the ListFindings and GroupFindings API ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) -* **securitycenter:** Added mute related APIs, proto messages and fields ([3e7185c](https://www.github.com/googleapis/google-cloud-go/commit/3e7185c241d97ee342f132ae04bc93bb79a8e897)) -* **securitycenter:** Added resource type and display_name field to the FindingResult, and supported them in the filter for ListFindings and GroupFindings. Also added display_name to the resource which is surfaced in NotificationMessage ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) -* **securitycenter:** Added vulnerability field to the finding feat: Added type field to the resource which is surfaced in NotificationMessage ([090cc3a](https://www.github.com/googleapis/google-cloud-go/commit/090cc3ae0f8747a14cc904fc6d429e2f5379bb03)) -* **servicecontrol:** add C++ rules for many Cloud services ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) -* **speech:** add result_end_time to SpeechRecognitionResult ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) -* **speech:** added alternative_language_codes to RecognitionConfig feat: WEBM_OPUS codec feat: SpeechAdaptation configuration feat: word confidence feat: spoken punctuation and spoken emojis feat: hint boost in SpeechContext ([a2c0bef](https://www.github.com/googleapis/google-cloud-go/commit/a2c0bef551489c9f1d0d12b973d3bf095354841e)) -* **texttospeech:** update v1 proto ([90e2868](https://www.github.com/googleapis/google-cloud-go/commit/90e2868a3d220aa7f897438f4917013fda7a7c59)) -* **workflows/executions:** add a stack_trace field to the Error messages specifying where the error occured feat: add call_log_level field to Execution messages doc: clarify requirement to escape strings within JSON arguments ([1f5aa78](https://www.github.com/googleapis/google-cloud-go/commit/1f5aa78a4d6633871651c89a6d9c48e3409fecc5)) - - -### Bug Fixes - -* **accesscontextmanager:** nodejs package name access-context-manager ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) -* **aiplatform:** Remove invalid resource annotations ([587bba5](https://www.github.com/googleapis/google-cloud-go/commit/587bba5ad792a92f252107aa38c6af50fb09fb58)) -* **compute/metadata:** return an error when all retries have failed ([#5063](https://www.github.com/googleapis/google-cloud-go/issues/5063)) ([c792a0d](https://www.github.com/googleapis/google-cloud-go/commit/c792a0d13db019c9964efeee5c6bc85b07ca50fa)), refs [#5062](https://www.github.com/googleapis/google-cloud-go/issues/5062) -* **compute:** make parent_id fields required compute move and insert methods ([#686](https://www.github.com/googleapis/google-cloud-go/issues/686)) ([c8271d4](https://www.github.com/googleapis/google-cloud-go/commit/c8271d4b217a6e6924d9f87eac9468c4b5767ba7)) -* **compute:** Move compute_small protos under its own directory ([#681](https://www.github.com/googleapis/google-cloud-go/issues/681)) ([3e7185c](https://www.github.com/googleapis/google-cloud-go/commit/3e7185c241d97ee342f132ae04bc93bb79a8e897)) -* **internal/gapicgen:** fix a compute filtering ([#5111](https://www.github.com/googleapis/google-cloud-go/issues/5111)) ([77aa19d](https://www.github.com/googleapis/google-cloud-go/commit/77aa19de7fc33a9e831e6b91bd324d6832b44d99)) -* **internal/godocfx:** only put TOC status on mod if all pkgs have same status ([#4974](https://www.github.com/googleapis/google-cloud-go/issues/4974)) ([309b59e](https://www.github.com/googleapis/google-cloud-go/commit/309b59e583d1bf0dd9ffe84223034eb8a2975d47)) -* **internal/godocfx:** replace * with HTML code ([#5049](https://www.github.com/googleapis/google-cloud-go/issues/5049)) ([a8f7c06](https://www.github.com/googleapis/google-cloud-go/commit/a8f7c066e8d97120ae4e12963e3c9acc8b8906c2)) -* **monitoring/apiv3:** Reintroduce deprecated field/enum for backward compatibility docs: Use absolute link targets in comments ([45fd259](https://www.github.com/googleapis/google-cloud-go/commit/45fd2594d99ef70c776df26866f0a3b537e7e69e)) -* **profiler:** workaround certificate expiration issue in integration tests ([#4955](https://www.github.com/googleapis/google-cloud-go/issues/4955)) ([de9e465](https://www.github.com/googleapis/google-cloud-go/commit/de9e465bea8cd0580c45e87d2cbc2b610615b363)) -* **security/privateca:** include mixin protos as input for mixin rpcs ([479c2f9](https://www.github.com/googleapis/google-cloud-go/commit/479c2f90d556a106b25ebcdb1539d231488182da)) -* **security/privateca:** repair service config to enable mixins ([83b941c](https://www.github.com/googleapis/google-cloud-go/commit/83b941c0983e44fdd18ceee8c6f3e91219d72ad1)) -* **video/transcoder:** update nodejs package name to video-transcoder ([30794e7](https://www.github.com/googleapis/google-cloud-go/commit/30794e70050b55ff87d6a80d0b4075065e9d271d)) - -## [0.97.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.96.0...v0.97.0) (2021-09-29) - - -### Features - -* **internal** add Retry func to testutil from samples repository [#4902](https://github.com/googleapis/google-cloud-go/pull/4902) - -## [0.96.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.95.0...v0.96.0) (2021-09-28) - - -### Features - -* **civil:** add IsEmpty function to time, date and datetime ([#4728](https://www.github.com/googleapis/google-cloud-go/issues/4728)) ([88bfa64](https://www.github.com/googleapis/google-cloud-go/commit/88bfa64d6df2f3bb7d41e0b8f56717dd3de790e2)), refs [#4727](https://www.github.com/googleapis/google-cloud-go/issues/4727) -* **internal/godocfx:** detect preview versions ([#4899](https://www.github.com/googleapis/google-cloud-go/issues/4899)) ([9b60844](https://www.github.com/googleapis/google-cloud-go/commit/9b608445ce9ebabbc87a50e85ce6ef89125031d2)) -* **internal:** provide wrapping for retried errors ([#4797](https://www.github.com/googleapis/google-cloud-go/issues/4797)) ([ce5f4db](https://www.github.com/googleapis/google-cloud-go/commit/ce5f4dbab884e847a2d9f1f8f3fcfd7df19a505a)) - - -### Bug Fixes - -* **internal/gapicgen:** restore fmting proto files ([#4789](https://www.github.com/googleapis/google-cloud-go/issues/4789)) ([5606b54](https://www.github.com/googleapis/google-cloud-go/commit/5606b54b97bb675487c6c138a4081c827218f933)) -* **internal/trace:** use xerrors.As for trace ([#4813](https://www.github.com/googleapis/google-cloud-go/issues/4813)) ([05fe61c](https://www.github.com/googleapis/google-cloud-go/commit/05fe61c5aa4860bdebbbe3e91a9afaba16aa6184)) - -## [0.95.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.94.1...v0.95.0) (2021-09-21) - -### Bug Fixes - -* **internal/gapicgen:** add a temporary import ([#4756](https://www.github.com/googleapis/google-cloud-go/issues/4756)) ([4d9c046](https://www.github.com/googleapis/google-cloud-go/commit/4d9c046b66a2dc205e2c14b676995771301440da)) -* **compute/metadata:** remove heavy gax dependency ([#4784](https://www.github.com/googleapis/google-cloud-go/issues/4784)) ([ea00264](https://www.github.com/googleapis/google-cloud-go/commit/ea00264428137471805f2ec67f04f3a5a42928fa)) - -### [0.94.1](https://www.github.com/googleapis/google-cloud-go/compare/v0.94.0...v0.94.1) (2021-09-02) - - -### Bug Fixes - -* **compute/metadata:** fix retry logic to not panic on error ([#4714](https://www.github.com/googleapis/google-cloud-go/issues/4714)) ([75c63b9](https://www.github.com/googleapis/google-cloud-go/commit/75c63b94d2cf86606fffc3611f7e6150b667eedc)), refs [#4713](https://www.github.com/googleapis/google-cloud-go/issues/4713) - -## [0.94.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.92.0...v0.94.0) (2021-08-31) - - -### Features - -* **aiplatform:** add XAI, model monitoring, and index services to aiplatform v1 ([e385b40](https://www.github.com/googleapis/google-cloud-go/commit/e385b40a1e2ecf81f5fd0910de5c37275951f86b)) -* **analytics/admin:** add `GetDataRetentionSettings`, `UpdateDataRetentionSettings` methods to the API ([8467899](https://www.github.com/googleapis/google-cloud-go/commit/8467899ab6ebf0328c543bfb5fbcddeb2f53a082)) -* **asset:** Release of relationships in v1, Add content type Relationship to support relationship export Committer: lvv@ ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) -* **assuredworkloads:** Add Canada Regions And Support compliance regime ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) -* **cloudbuild/apiv1:** Add ability to configure BuildTriggers to create Builds that require approval before executing and ApproveBuild API to approve or reject pending Builds ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) -* **cloudbuild/apiv1:** add script field to BuildStep message ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) -* **cloudbuild/apiv1:** Update cloudbuild proto with the service_account for BYOSA Triggers. ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) -* **compute/metadata:** retry error when talking to metadata service ([#4648](https://www.github.com/googleapis/google-cloud-go/issues/4648)) ([81c6039](https://www.github.com/googleapis/google-cloud-go/commit/81c6039503121f8da3de4f4cd957b8488a3ef620)), refs [#4642](https://www.github.com/googleapis/google-cloud-go/issues/4642) -* **dataproc:** remove apiv1beta2 client ([#4682](https://www.github.com/googleapis/google-cloud-go/issues/4682)) ([2248554](https://www.github.com/googleapis/google-cloud-go/commit/22485541affb1251604df292670a20e794111d3e)) -* **gaming:** support version reporting API ([cd65cec](https://www.github.com/googleapis/google-cloud-go/commit/cd65cecf15c4a01648da7f8f4f4d497772961510)) -* **gkehub:** Add request_id under `DeleteMembershipRequest` and `UpdateMembershipRequest` ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) -* **internal/carver:** support carving batches ([#4623](https://www.github.com/googleapis/google-cloud-go/issues/4623)) ([2972d19](https://www.github.com/googleapis/google-cloud-go/commit/2972d194da19bedf16d76fda471c06a965cfdcd6)) -* **kms:** add support for Key Reimport ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) -* **metastore:** Added the Backup resource and Backup resource GetIamPolicy/SetIamPolicy to V1 feat: Added the RestoreService method to V1 ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) -* **monitoring/dashboard:** Added support for logs-based alerts: https://cloud.google.com/logging/docs/alerting/log-based-alerts feat: Added support for user-defined labels on cloud monitoring's Service and ServiceLevelObjective objects fix!: mark required fields in QueryTimeSeriesRequest as required ([b9226eb](https://www.github.com/googleapis/google-cloud-go/commit/b9226eb0b34473cb6f920c2526ad0d6dacb03f3c)) -* **osconfig:** Update osconfig v1 and v1alpha with WindowsApplication ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) -* **speech:** Add transcript normalization ([b31646d](https://www.github.com/googleapis/google-cloud-go/commit/b31646d1e12037731df4b5c0ba9f60b6434d7b9b)) -* **talent:** Add new commute methods in Search APIs feat: Add new histogram type 'publish_time_in_day' feat: Support filtering by requisitionId is ListJobs API ([d4c3340](https://www.github.com/googleapis/google-cloud-go/commit/d4c3340bfc8b6793d6d2c8a3ed8ccdb472e1efd3)) -* **translate:** added v3 proto for online/batch document translation and updated v3beta1 proto for format conversion ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) - - -### Bug Fixes - -* **datastream:** Change a few resource pattern variables from camelCase to snake_case ([bf4378b](https://www.github.com/googleapis/google-cloud-go/commit/bf4378b5b859f7b835946891dbfebfee31c4b123)) - -## [0.92.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.91.0...v0.92.0) (2021-08-16) - - -### Features - -* **all:** remove testing deps ([#4580](https://www.github.com/googleapis/google-cloud-go/issues/4580)) ([15c1eb9](https://www.github.com/googleapis/google-cloud-go/commit/15c1eb9730f0b514edb911161f9c59e8d790a5ec)), refs [#4061](https://www.github.com/googleapis/google-cloud-go/issues/4061) -* **internal/detect:** add helper to detect projectID from env ([#4582](https://www.github.com/googleapis/google-cloud-go/issues/4582)) ([cc65d94](https://www.github.com/googleapis/google-cloud-go/commit/cc65d945688ac446602bce6ef86a935714dfe2f8)), refs [#1294](https://www.github.com/googleapis/google-cloud-go/issues/1294) -* **spannertest:** Add validation of duplicated column names ([#4611](https://www.github.com/googleapis/google-cloud-go/issues/4611)) ([84f86a6](https://www.github.com/googleapis/google-cloud-go/commit/84f86a605c809ab36dd3cb4b3ab1df15a5302083)) - -## [0.91.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.90.0...v0.91.0) (2021-08-11) - - -### Features - -* **.github:** support dynamic submodule detection ([#4537](https://www.github.com/googleapis/google-cloud-go/issues/4537)) ([4374b90](https://www.github.com/googleapis/google-cloud-go/commit/4374b907e9f166da6bd23a8ef94399872b00afd6)) -* **dialogflow/cx:** add advanced settings for agent level feat: add rollout config, state and failure reason for experiment feat: add insights export settings for security setting feat: add language code for streaming recognition result and flow versions for query parameters docs: deprecate legacy logging settings ([ed73554](https://www.github.com/googleapis/google-cloud-go/commit/ed735541dc57d0681d84b46853393eac5f7ccec3)) -* **dialogflow/cx:** add advanced settings for agent level feat: add rollout config, state and failure reason for experiment feat: add insights export settings for security setting feat: add language code for streaming recognition result and flow versions for query parameters docs: deprecate legacy logging settings ([ed73554](https://www.github.com/googleapis/google-cloud-go/commit/ed735541dc57d0681d84b46853393eac5f7ccec3)) -* **dialogflow/cx:** added support for DLP templates; expose `Locations` service to get/list avaliable locations of Dialogflow products ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) -* **dialogflow/cx:** added support for DLP templates; expose `Locations` service to get/list avaliable locations of Dialogflow products docs: reorder some fields ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) -* **dialogflow:** expose `Locations` service to get/list avaliable locations of Dialogflow products; fixed some API annotations ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) -* **kms:** add support for HMAC, Variable Key Destruction, and GenerateRandom ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) -* **speech:** add total_billed_time response field ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) -* **video/transcoder:** Add video cropping feature feat: Add video padding feature feat: Add ttl_after_completion_days field to Job docs: Update proto documentation docs: Indicate v1beta1 deprecation ([5996846](https://www.github.com/googleapis/google-cloud-go/commit/59968462a3870c6289166fa1161f9b6d9c10e093)) - - -### Bug Fixes - -* **functions:** Updating behavior of source_upload_url during Get/List function calls ([381a494](https://www.github.com/googleapis/google-cloud-go/commit/381a494c29da388977b0bdda2177058328cc4afe)) - -## [0.90.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.89.0...v0.90.0) (2021-08-03) - - -### ⚠ BREAKING CHANGES - -* **compute:** add pagination and an Operation wrapper (#4542) - -### Features - -* **compute:** add pagination and an Operation wrapper ([#4542](https://www.github.com/googleapis/google-cloud-go/issues/4542)) ([36f4649](https://www.github.com/googleapis/google-cloud-go/commit/36f46494111f6d16d103fb208d49616576dbf91e)) -* **internal/godocfx:** add status to packages and TOCs ([#4547](https://www.github.com/googleapis/google-cloud-go/issues/4547)) ([c6de69c](https://www.github.com/googleapis/google-cloud-go/commit/c6de69c710561bb2a40eff05417df4b9798c258a)) -* **internal/godocfx:** mark status of deprecated items ([#4525](https://www.github.com/googleapis/google-cloud-go/issues/4525)) ([d571c6f](https://www.github.com/googleapis/google-cloud-go/commit/d571c6f4337ec9c4807c230cd77f53b6e7db6437)) - - -### Bug Fixes - -* **internal/carver:** don't tag commits ([#4518](https://www.github.com/googleapis/google-cloud-go/issues/4518)) ([c355eb8](https://www.github.com/googleapis/google-cloud-go/commit/c355eb8ecb0bb1af0ccf55e6262ca8c0d5c7e352)) - -## [0.89.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.88.0...v0.89.0) (2021-07-29) - - -### Features - -* **assuredworkloads:** Add EU Regions And Support compliance regime ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **datacatalog:** Added support for BigQuery connections entries feat: Added support for BigQuery routines entries feat: Added usage_signal field feat: Added labels field feat: Added ReplaceTaxonomy in Policy Tag Manager Serialization API feat: Added support for public tag templates feat: Added support for rich text tags docs: Documentation improvements ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **datafusion:** start generating apiv1 ([e55a016](https://www.github.com/googleapis/google-cloud-go/commit/e55a01667afaf36ff70807d061ecafb61827ba97)) -* **iap:** start generating apiv1 ([e55a016](https://www.github.com/googleapis/google-cloud-go/commit/e55a01667afaf36ff70807d061ecafb61827ba97)) -* **internal/carver:** add tooling to help carve out sub-modules ([#4417](https://www.github.com/googleapis/google-cloud-go/issues/4417)) ([a7e28f2](https://www.github.com/googleapis/google-cloud-go/commit/a7e28f2557469562ae57e5174b41bdf8fce62b63)) -* **networkconnectivity:** Add files for Network Connectivity v1 API. ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **retail:** Add restricted Retail Search features for Retail API v2. ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **secretmanager:** In Secret Manager, users can now use filter to customize the output of ListSecrets/ListSecretVersions calls ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **securitycenter:** add finding_class and indicator fields in Finding ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **speech:** add total_billed_time response field. fix!: phrase_set_id is required field in CreatePhraseSetRequest. fix!: custom_class_id is required field in CreateCustomClassRequest. ([a52baa4](https://www.github.com/googleapis/google-cloud-go/commit/a52baa456ed8513ec492c4b573c191eb61468758)) -* **storagetransfer:** start generating apiv1 ([#4505](https://www.github.com/googleapis/google-cloud-go/issues/4505)) ([f2d531d](https://www.github.com/googleapis/google-cloud-go/commit/f2d531d2b519efa58e0f23a178bbebe675c203c3)) - - -### Bug Fixes - -* **internal/gapicgen:** exec Stdout already set ([#4509](https://www.github.com/googleapis/google-cloud-go/issues/4509)) ([41246e9](https://www.github.com/googleapis/google-cloud-go/commit/41246e900aaaea92a9f956e92956c40c86f4cb3a)) -* **internal/gapicgen:** tidy all after dep bump ([#4515](https://www.github.com/googleapis/google-cloud-go/issues/4515)) ([9401be5](https://www.github.com/googleapis/google-cloud-go/commit/9401be509c570c3c55694375065c84139e961857)), refs [#4434](https://www.github.com/googleapis/google-cloud-go/issues/4434) - -## [0.88.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.87.0...v0.88.0) (2021-07-22) - - -### ⚠ BREAKING CHANGES - -* **cloudbuild/apiv1:** Proto had a prior definitions of WorkerPool resources which were never supported. This change replaces those resources with definitions that are currently supported. - -### Features - -* **cloudbuild/apiv1:** add a WorkerPools API ([19ea3f8](https://www.github.com/googleapis/google-cloud-go/commit/19ea3f830212582bfee21d9e09f0034f9ce76547)) -* **cloudbuild/apiv1:** Implementation of Build Failure Info: - Added message FailureInfo field ([19ea3f8](https://www.github.com/googleapis/google-cloud-go/commit/19ea3f830212582bfee21d9e09f0034f9ce76547)) -* **osconfig/agentendpoint:** OSConfig AgentEndpoint: add basic os info to RegisterAgentRequest, add WindowsApplication type to Inventory ([8936bc3](https://www.github.com/googleapis/google-cloud-go/commit/8936bc3f2d0fb2f6514f6e019fa247b8f41bd43c)) -* **resourcesettings:** Publish Cloud ResourceSettings v1 API ([43ad3cb](https://www.github.com/googleapis/google-cloud-go/commit/43ad3cb7be981fff9dc5dcf4510f1cd7bea99957)) - - -### Bug Fixes - -* **internal/godocfx:** set exit code, print cmd output, no go get ... ([#4445](https://www.github.com/googleapis/google-cloud-go/issues/4445)) ([cc70f77](https://www.github.com/googleapis/google-cloud-go/commit/cc70f77ac279a62e24e1b07f6e53fd126b7286b0)) -* **internal:** detect module for properly generating docs URLs ([#4460](https://www.github.com/googleapis/google-cloud-go/issues/4460)) ([1eaba8b](https://www.github.com/googleapis/google-cloud-go/commit/1eaba8bd694f7552a8e3e09b4f164de8b6ca23f0)), refs [#4447](https://www.github.com/googleapis/google-cloud-go/issues/4447) -* **kms:** Updating WORKSPACE files to use the newest version of the Typescript generator. ([8936bc3](https://www.github.com/googleapis/google-cloud-go/commit/8936bc3f2d0fb2f6514f6e019fa247b8f41bd43c)) - -## [0.87.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.86.0...v0.87.0) (2021-07-13) - - -### Features - -* **container:** allow updating security group on existing clusters ([528ffc9](https://www.github.com/googleapis/google-cloud-go/commit/528ffc9bd63090129a8b1355cd31273f8c23e34c)) -* **monitoring/dashboard:** added validation only mode when writing dashboards feat: added alert chart widget ([652d7c2](https://www.github.com/googleapis/google-cloud-go/commit/652d7c277da2f6774729064ab65d557875c81567)) -* **networkmanagment:** start generating apiv1 ([907592c](https://www.github.com/googleapis/google-cloud-go/commit/907592c576abfc65c01bbcd30c1a6094916cdc06)) -* **secretmanager:** Tune Secret Manager auto retry parameters ([528ffc9](https://www.github.com/googleapis/google-cloud-go/commit/528ffc9bd63090129a8b1355cd31273f8c23e34c)) -* **video/transcoder:** start generating apiv1 ([907592c](https://www.github.com/googleapis/google-cloud-go/commit/907592c576abfc65c01bbcd30c1a6094916cdc06)) - - -### Bug Fixes - -* **compute:** properly generate PUT requests ([#4426](https://www.github.com/googleapis/google-cloud-go/issues/4426)) ([a7491a5](https://www.github.com/googleapis/google-cloud-go/commit/a7491a533e4ad75eb6d5f89718d4dafb0c5b4167)) -* **internal:** fix relative pathing for generator ([#4397](https://www.github.com/googleapis/google-cloud-go/issues/4397)) ([25e0eae](https://www.github.com/googleapis/google-cloud-go/commit/25e0eaecf9feb1caa97988c5398ac58f6ca17391)) - - -### Miscellaneous Chores - -* **all:** fix release version ([#4427](https://www.github.com/googleapis/google-cloud-go/issues/4427)) ([2c0d267](https://www.github.com/googleapis/google-cloud-go/commit/2c0d2673ccab7281b6432215ee8279f9efd04a15)) - -## [0.86.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.85.0...v0.86.0) (2021-07-01) - - -### Features - -* **bigquery managedwriter:** schema conversion support ([#4357](https://www.github.com/googleapis/google-cloud-go/issues/4357)) ([f2b20f4](https://www.github.com/googleapis/google-cloud-go/commit/f2b20f493e2ed5a883ce42fa65695c03c574feb5)) - -## [0.85.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.84.0...v0.85.0) (2021-06-30) - - -### Features - -* **dataflow:** start generating apiv1beta3 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) -* **datastream:** start generating apiv1alpha1 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) -* **dialogflow:** added Automated agent reply type and allow cancellation flag for partial response feature. ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) -* **documentai:** update document.proto, add the processor management methods. ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) -* **eventarc:** start generating apiv1 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) -* **gkehub:** added v1alpha messages and client for gkehub ([8fb4649](https://www.github.com/googleapis/google-cloud-go/commit/8fb464956f0ca51d30e8e14dc625ff9fa150c437)) -* **internal/godocfx:** add support for other modules ([#4290](https://www.github.com/googleapis/google-cloud-go/issues/4290)) ([d52bae6](https://www.github.com/googleapis/google-cloud-go/commit/d52bae6cd77474174192c46236d309bf967dfa00)) -* **internal/godocfx:** different metadata for different modules ([#4297](https://www.github.com/googleapis/google-cloud-go/issues/4297)) ([598f5b9](https://www.github.com/googleapis/google-cloud-go/commit/598f5b93778b2e2e75265ae54484dd54477433f5)) -* **internal:** add force option for regen ([#4310](https://www.github.com/googleapis/google-cloud-go/issues/4310)) ([de654eb](https://www.github.com/googleapis/google-cloud-go/commit/de654ebfcf23a53b4d1fee0aa48c73999a55c1a6)) -* **servicecontrol:** Added the gRPC service config for the Service Controller v1 API docs: Updated some comments. ([8fb4649](https://www.github.com/googleapis/google-cloud-go/commit/8fb464956f0ca51d30e8e14dc625ff9fa150c437)) -* **workflows/executions:** start generating apiv1 ([cfee361](https://www.github.com/googleapis/google-cloud-go/commit/cfee36161d41e3a0f769e51ab96c25d0967af273)) - - -### Bug Fixes - -* **internal:** add autogenerated header to snippets ([#4261](https://www.github.com/googleapis/google-cloud-go/issues/4261)) ([2220787](https://www.github.com/googleapis/google-cloud-go/commit/222078722c37c3fdadec7bbbe0bcf81edd105f1a)), refs [#4260](https://www.github.com/googleapis/google-cloud-go/issues/4260) -* **internal:** fix googleapis-disco regen ([#4354](https://www.github.com/googleapis/google-cloud-go/issues/4354)) ([aeea1ce](https://www.github.com/googleapis/google-cloud-go/commit/aeea1ce1e5dff3acdfe208932327b52c49851b41)) -* **kms:** replace IAMPolicy mixin in service config. ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) -* **security/privateca:** Fixed casing of the Ruby namespace ([5a9c6ce](https://www.github.com/googleapis/google-cloud-go/commit/5a9c6ce781fb6a338e29d3dee72367998d834af0)) - -## [0.84.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.83.0...v0.84.0) (2021-06-09) - - -### Features - -* **aiplatform:** start generating apiv1 ([be1d729](https://www.github.com/googleapis/google-cloud-go/commit/be1d729fdaa18eb1c782f3b09a6bb8fd6b3a144c)) -* **apigeeconnect:** start generating abiv1 ([be1d729](https://www.github.com/googleapis/google-cloud-go/commit/be1d729fdaa18eb1c782f3b09a6bb8fd6b3a144c)) -* **dialogflow/cx:** support sentiment analysis in bot testing ([7a57aac](https://www.github.com/googleapis/google-cloud-go/commit/7a57aac996f2bae20ee6ddbd02ad9e56e380099b)) -* **dialogflow/cx:** support sentiment analysis in bot testing ([6ad2306](https://www.github.com/googleapis/google-cloud-go/commit/6ad2306f64710ce16059b464342dbc6a98d2d9c2)) -* **documentai:** Move CommonOperationMetadata into a separate proto file for potential reuse. ([9e80ea0](https://www.github.com/googleapis/google-cloud-go/commit/9e80ea0d053b06876418194f65a478045dc4fe6c)) -* **documentai:** Move CommonOperationMetadata into a separate proto file for potential reuse. ([18375e5](https://www.github.com/googleapis/google-cloud-go/commit/18375e50e8f16e63506129b8927a7b62f85e407b)) -* **gkeconnect/gateway:** start generating apiv1beta1 ([#4235](https://www.github.com/googleapis/google-cloud-go/issues/4235)) ([1c3e968](https://www.github.com/googleapis/google-cloud-go/commit/1c3e9689d78670a231a3660db00fd4fd8f5c6345)) -* **lifesciences:** strat generating apiv2beta ([be1d729](https://www.github.com/googleapis/google-cloud-go/commit/be1d729fdaa18eb1c782f3b09a6bb8fd6b3a144c)) -* **tpu:** start generating apiv1 ([#4199](https://www.github.com/googleapis/google-cloud-go/issues/4199)) ([cac48ea](https://www.github.com/googleapis/google-cloud-go/commit/cac48eab960cd34cc20732f6a1aeb93c540a036b)) - - -### Bug Fixes - -* **bttest:** fix race condition in SampleRowKeys ([#4207](https://www.github.com/googleapis/google-cloud-go/issues/4207)) ([5711fb1](https://www.github.com/googleapis/google-cloud-go/commit/5711fb10d25c458807598d736a232bb2210a047a)) -* **documentai:** Fix Ruby gem title of documentai v1 (package not currently published) ([9e80ea0](https://www.github.com/googleapis/google-cloud-go/commit/9e80ea0d053b06876418194f65a478045dc4fe6c)) - -## [0.83.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.82.0...v0.83.0) (2021-06-02) - - -### Features - -* **dialogflow:** added a field in the query result to indicate whether slot filling is cancelled. ([f9cda8f](https://www.github.com/googleapis/google-cloud-go/commit/f9cda8fb6c3d76a062affebe6649f0a43aeb96f3)) -* **essentialcontacts:** start generating apiv1 ([#4118](https://www.github.com/googleapis/google-cloud-go/issues/4118)) ([fe14afc](https://www.github.com/googleapis/google-cloud-go/commit/fe14afcf74e09089b22c4f5221cbe37046570fda)) -* **gsuiteaddons:** start generating apiv1 ([#4082](https://www.github.com/googleapis/google-cloud-go/issues/4082)) ([6de5c99](https://www.github.com/googleapis/google-cloud-go/commit/6de5c99173c4eeaf777af18c47522ca15637d232)) -* **osconfig:** OSConfig: add ExecResourceOutput and per step error message. ([f9cda8f](https://www.github.com/googleapis/google-cloud-go/commit/f9cda8fb6c3d76a062affebe6649f0a43aeb96f3)) -* **osconfig:** start generating apiv1alpha ([#4119](https://www.github.com/googleapis/google-cloud-go/issues/4119)) ([8ad471f](https://www.github.com/googleapis/google-cloud-go/commit/8ad471f26087ec076460df6dcf27769ffe1b8834)) -* **privatecatalog:** start generating apiv1beta1 ([500c1a6](https://www.github.com/googleapis/google-cloud-go/commit/500c1a6101f624cb6032f0ea16147645a02e7076)) -* **serviceusage:** start generating apiv1 ([#4120](https://www.github.com/googleapis/google-cloud-go/issues/4120)) ([e4531f9](https://www.github.com/googleapis/google-cloud-go/commit/e4531f93cfeb6388280bb253ef6eb231aba37098)) -* **shell:** start generating apiv1 ([500c1a6](https://www.github.com/googleapis/google-cloud-go/commit/500c1a6101f624cb6032f0ea16147645a02e7076)) -* **vpcaccess:** start generating apiv1 ([500c1a6](https://www.github.com/googleapis/google-cloud-go/commit/500c1a6101f624cb6032f0ea16147645a02e7076)) - -## [0.82.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.81.0...v0.82.0) (2021-05-17) - - -### Features - -* **billing/budgets:** Added support for configurable budget time period. fix: Updated some documentation links. ([83b1b3b](https://www.github.com/googleapis/google-cloud-go/commit/83b1b3b648c6d9225f07f00e8c0cdabc3d1fc1ab)) -* **billing/budgets:** Added support for configurable budget time period. fix: Updated some documentation links. ([83b1b3b](https://www.github.com/googleapis/google-cloud-go/commit/83b1b3b648c6d9225f07f00e8c0cdabc3d1fc1ab)) -* **cloudbuild/apiv1:** Add fields for Pub/Sub triggers ([8b4adbf](https://www.github.com/googleapis/google-cloud-go/commit/8b4adbf9815e1ec229dfbcfb9189d3ea63112e1b)) -* **cloudbuild/apiv1:** Implementation of Source Manifests: - Added message StorageSourceManifest as an option for the Source message - Added StorageSourceManifest field to the SourceProvenance message ([7fd2ccd](https://www.github.com/googleapis/google-cloud-go/commit/7fd2ccd26adec1468e15fe84bf75210255a9dfea)) -* **clouddms:** start generating apiv1 ([#4081](https://www.github.com/googleapis/google-cloud-go/issues/4081)) ([29df85c](https://www.github.com/googleapis/google-cloud-go/commit/29df85c40ab64d59e389a980c9ce550077839763)) -* **dataproc:** update the Dataproc V1 API client library ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) -* **dialogflow/cx:** add support for service directory webhooks ([7fd2ccd](https://www.github.com/googleapis/google-cloud-go/commit/7fd2ccd26adec1468e15fe84bf75210255a9dfea)) -* **dialogflow/cx:** add support for service directory webhooks ([7fd2ccd](https://www.github.com/googleapis/google-cloud-go/commit/7fd2ccd26adec1468e15fe84bf75210255a9dfea)) -* **dialogflow/cx:** support setting current_page to resume sessions; expose transition_route_groups in flows and language_code in webhook ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) -* **dialogflow/cx:** support setting current_page to resume sessions; expose transition_route_groups in flows and language_code in webhook ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) -* **dialogflow:** added more Environment RPCs feat: added Versions service feat: added Fulfillment service feat: added TextToSpeechSettings. feat: added location in some resource patterns. ([4f73dc1](https://www.github.com/googleapis/google-cloud-go/commit/4f73dc19c2e05ad6133a8eac3d62ddb522314540)) -* **documentai:** add confidence field to the PageAnchor.PageRef in document.proto. ([d089dda](https://www.github.com/googleapis/google-cloud-go/commit/d089dda0089acb9aaef9b3da40b219476af9fc06)) -* **documentai:** add confidence field to the PageAnchor.PageRef in document.proto. ([07fdcd1](https://www.github.com/googleapis/google-cloud-go/commit/07fdcd12499eac26f9b5fae01d6c1282c3e02b7c)) -* **internal/gapicgen:** only update relevant gapic files ([#4066](https://www.github.com/googleapis/google-cloud-go/issues/4066)) ([5948bee](https://www.github.com/googleapis/google-cloud-go/commit/5948beedbadd491601bdee6a006cf685e94a85f4)) -* **internal/gensnippets:** add license header and region tags ([#3924](https://www.github.com/googleapis/google-cloud-go/issues/3924)) ([e9ff7a0](https://www.github.com/googleapis/google-cloud-go/commit/e9ff7a0f9bb1cc67f5d0de47934811960429e72c)) -* **internal/gensnippets:** initial commit ([#3922](https://www.github.com/googleapis/google-cloud-go/issues/3922)) ([3fabef0](https://www.github.com/googleapis/google-cloud-go/commit/3fabef032388713f732ab4dbfc51624cdca0f481)) -* **internal:** auto-generate snippets ([#3949](https://www.github.com/googleapis/google-cloud-go/issues/3949)) ([b70e0fc](https://www.github.com/googleapis/google-cloud-go/commit/b70e0fccdc86813e0d97ff63b585822d4deafb38)) -* **internal:** generate region tags for snippets ([#3962](https://www.github.com/googleapis/google-cloud-go/issues/3962)) ([ef2b90e](https://www.github.com/googleapis/google-cloud-go/commit/ef2b90ea6d47e27744c98a1a9ae0c487c5051808)) -* **metastore:** start generateing apiv1 ([#4083](https://www.github.com/googleapis/google-cloud-go/issues/4083)) ([661610a](https://www.github.com/googleapis/google-cloud-go/commit/661610afa6a9113534884cafb138109536724310)) -* **security/privateca:** start generating apiv1 ([#4023](https://www.github.com/googleapis/google-cloud-go/issues/4023)) ([08aa83a](https://www.github.com/googleapis/google-cloud-go/commit/08aa83a5371bb6485bc3b19b3ed5300f807ce69f)) -* **securitycenter:** add canonical_name and folder fields ([5c5ca08](https://www.github.com/googleapis/google-cloud-go/commit/5c5ca08c637a23cfa3e3a051fea576e1feb324fd)) -* **securitycenter:** add canonical_name and folder fields ([5c5ca08](https://www.github.com/googleapis/google-cloud-go/commit/5c5ca08c637a23cfa3e3a051fea576e1feb324fd)) -* **speech:** add webm opus support. ([d089dda](https://www.github.com/googleapis/google-cloud-go/commit/d089dda0089acb9aaef9b3da40b219476af9fc06)) -* **speech:** Support for spoken punctuation and spoken emojis. ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) - - -### Bug Fixes - -* **binaryauthorization:** add Java options to Binaryauthorization protos ([9a459d5](https://www.github.com/googleapis/google-cloud-go/commit/9a459d5d149b9c3b02a35d4245d164b899ff09b3)) -* **internal/gapicgen:** filter out internal directory changes ([#4085](https://www.github.com/googleapis/google-cloud-go/issues/4085)) ([01473f6](https://www.github.com/googleapis/google-cloud-go/commit/01473f6d8db26c6e18969ace7f9e87c66e94ad9e)) -* **internal/gapicgen:** use correct region tags for gensnippets ([#4022](https://www.github.com/googleapis/google-cloud-go/issues/4022)) ([8ccd689](https://www.github.com/googleapis/google-cloud-go/commit/8ccd689cab08f016008ca06a939a4828817d4a25)) -* **internal/gensnippets:** run goimports ([#3931](https://www.github.com/googleapis/google-cloud-go/issues/3931)) ([10050f0](https://www.github.com/googleapis/google-cloud-go/commit/10050f05c20c226547d87c08168fa4bc551395c5)) -* **internal:** append a new line to comply with go fmt ([#4028](https://www.github.com/googleapis/google-cloud-go/issues/4028)) ([a297278](https://www.github.com/googleapis/google-cloud-go/commit/a2972783c4af806199d1c67c9f63ad9677f20f34)) -* **internal:** make sure formatting is run on snippets ([#4039](https://www.github.com/googleapis/google-cloud-go/issues/4039)) ([130dfc5](https://www.github.com/googleapis/google-cloud-go/commit/130dfc535396e98fc009585b0457e3bc48ead941)), refs [#4037](https://www.github.com/googleapis/google-cloud-go/issues/4037) -* **metastore:** increase metastore lro polling timeouts ([83b1b3b](https://www.github.com/googleapis/google-cloud-go/commit/83b1b3b648c6d9225f07f00e8c0cdabc3d1fc1ab)) - - -### Miscellaneous Chores - -* **all:** fix release version ([#4040](https://www.github.com/googleapis/google-cloud-go/issues/4040)) ([4c991a9](https://www.github.com/googleapis/google-cloud-go/commit/4c991a928665d9be93691decce0c653f430688b7)) - -## [0.81.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.80.0...v0.81.0) (2021-04-02) - - -### Features - -* **datacatalog:** Policy Tag Manager v1 API service feat: new RenameTagTemplateFieldEnumValue API feat: adding fully_qualified_name in lookup and search feat: added DATAPROC_METASTORE integrated system along with new entry types: DATABASE and SERVICE docs: Documentation improvements ([2b02a03](https://www.github.com/googleapis/google-cloud-go/commit/2b02a03ff9f78884da5a8e7b64a336014c61bde7)) -* **dialogflow/cx:** include original user query in WebhookRequest; add GetTextCaseresult API. doc: clarify resource format for session response. ([a0b1f6f](https://www.github.com/googleapis/google-cloud-go/commit/a0b1f6faae77d014fdee166ab018ddcd6f846ab4)) -* **dialogflow/cx:** include original user query in WebhookRequest; add GetTextCaseresult API. doc: clarify resource format for session response. ([b5b4da6](https://www.github.com/googleapis/google-cloud-go/commit/b5b4da6952922440d03051f629f3166f731dfaa3)) -* **dialogflow:** expose MP3_64_KBPS and MULAW for output audio encodings. ([b5b4da6](https://www.github.com/googleapis/google-cloud-go/commit/b5b4da6952922440d03051f629f3166f731dfaa3)) -* **secretmanager:** Rotation for Secrets ([2b02a03](https://www.github.com/googleapis/google-cloud-go/commit/2b02a03ff9f78884da5a8e7b64a336014c61bde7)) - - -### Bug Fixes - -* **internal/godocfx:** filter out non-Cloud ([#3878](https://www.github.com/googleapis/google-cloud-go/issues/3878)) ([625aef9](https://www.github.com/googleapis/google-cloud-go/commit/625aef9b47181cf627587cc9cde9e400713c6678)) - -## [0.80.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.79.0...v0.80.0) (2021-03-23) - - -### ⚠ BREAKING CHANGES - -* **all:** This is a breaking change in dialogflow - -### Features - -* **appengine:** added vm_liveness, search_api_available, network_settings, service_account, build_env_variables, kms_key_reference to v1 API ([fd04a55](https://www.github.com/googleapis/google-cloud-go/commit/fd04a552213f99619c714b5858548f61f4948493)) -* **assuredworkloads:** Add 'resource_settings' field to provide custom properties (ids) for the provisioned projects. ([ab4824a](https://www.github.com/googleapis/google-cloud-go/commit/ab4824a7914864228e59b244d6382de862139524)) -* **assuredworkloads:** add HIPAA and HITRUST compliance regimes ([ab4824a](https://www.github.com/googleapis/google-cloud-go/commit/ab4824a7914864228e59b244d6382de862139524)) -* **dialogflow/cx:** added fallback option when restoring an agent docs: clarified experiment length ([cd70aa9](https://www.github.com/googleapis/google-cloud-go/commit/cd70aa9cc1a5dccfe4e49d2d6ca6db2119553c86)) -* **dialogflow/cx:** start generating apiv3 ([#3850](https://www.github.com/googleapis/google-cloud-go/issues/3850)) ([febbdcf](https://www.github.com/googleapis/google-cloud-go/commit/febbdcf13fcea3f5d8186c3d3dface1c0d27ef9e)), refs [#3634](https://www.github.com/googleapis/google-cloud-go/issues/3634) -* **documentai:** add EVAL_SKIPPED value to the Provenance.OperationType enum in document.proto. ([cb43066](https://www.github.com/googleapis/google-cloud-go/commit/cb4306683926843f6e977f207fa6070bb9242a61)) -* **documentai:** start generating apiv1 ([#3853](https://www.github.com/googleapis/google-cloud-go/issues/3853)) ([d68e604](https://www.github.com/googleapis/google-cloud-go/commit/d68e604c953eea90489f6134e71849b24dd0fcbf)) -* **internal/godocfx:** add prettyprint class to code blocks ([#3819](https://www.github.com/googleapis/google-cloud-go/issues/3819)) ([6e49f21](https://www.github.com/googleapis/google-cloud-go/commit/6e49f2148b116ee439c8a882dcfeefb6e7647c57)) -* **internal/godocfx:** handle Markdown content ([#3816](https://www.github.com/googleapis/google-cloud-go/issues/3816)) ([56d5d0a](https://www.github.com/googleapis/google-cloud-go/commit/56d5d0a900197fb2de46120a0eda649f2c17448f)) -* **kms:** Add maxAttempts to retry policy for KMS gRPC service config feat: Add Bazel exports_files entry for KMS gRPC service config ([fd04a55](https://www.github.com/googleapis/google-cloud-go/commit/fd04a552213f99619c714b5858548f61f4948493)) -* **resourcesettings:** start generating apiv1 ([#3854](https://www.github.com/googleapis/google-cloud-go/issues/3854)) ([3b288b4](https://www.github.com/googleapis/google-cloud-go/commit/3b288b4fa593c6cb418f696b5b26768967c20b9e)) -* **speech:** Support output transcript to GCS for LongRunningRecognize. ([fd04a55](https://www.github.com/googleapis/google-cloud-go/commit/fd04a552213f99619c714b5858548f61f4948493)) -* **speech:** Support output transcript to GCS for LongRunningRecognize. ([cd70aa9](https://www.github.com/googleapis/google-cloud-go/commit/cd70aa9cc1a5dccfe4e49d2d6ca6db2119553c86)) -* **speech:** Support output transcript to GCS for LongRunningRecognize. ([35a8706](https://www.github.com/googleapis/google-cloud-go/commit/35a870662df8bf63c4ec10a0233d1d7a708007ee)) - - -### Miscellaneous Chores - -* **all:** auto-regenerate gapics ([#3837](https://www.github.com/googleapis/google-cloud-go/issues/3837)) ([ab4824a](https://www.github.com/googleapis/google-cloud-go/commit/ab4824a7914864228e59b244d6382de862139524)) - -## [0.79.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.78.0...v0.79.0) (2021-03-10) - - -### Features - -* **apigateway:** start generating apiv1 ([#3726](https://www.github.com/googleapis/google-cloud-go/issues/3726)) ([66046da](https://www.github.com/googleapis/google-cloud-go/commit/66046da2a4be5971ce2655dc6a5e1fadb08c3d1f)) -* **channel:** addition of billing_account field on Plan. docs: clarification that valid address lines are required for all customers. ([d4246aa](https://www.github.com/googleapis/google-cloud-go/commit/d4246aad4da3c3ef12350385f229bb908e3fb215)) -* **dialogflow/cx:** allow to disable webhook invocation per request ([d4246aa](https://www.github.com/googleapis/google-cloud-go/commit/d4246aad4da3c3ef12350385f229bb908e3fb215)) -* **dialogflow/cx:** allow to disable webhook invocation per request ([44c6bf9](https://www.github.com/googleapis/google-cloud-go/commit/44c6bf986f39a3c9fddf46788ae63bfbb3739441)) -* **dialogflow:** Add CCAI API ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) -* **documentai:** remove the translation fields in document.proto. ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) -* **documentai:** Update documentai/v1beta3 protos: add support for boolean normalized value ([529925b](https://www.github.com/googleapis/google-cloud-go/commit/529925ba79f4d3191ef80a13e566d86210fe4d25)) -* **internal/godocfx:** keep some cross links on same domain ([#3767](https://www.github.com/googleapis/google-cloud-go/issues/3767)) ([77f76ed](https://www.github.com/googleapis/google-cloud-go/commit/77f76ed09cb07a090ba9054063a7c002a35bca4e)) -* **internal:** add ability to regenerate one module's docs ([#3777](https://www.github.com/googleapis/google-cloud-go/issues/3777)) ([dc15995](https://www.github.com/googleapis/google-cloud-go/commit/dc15995521bd065da4cfaae95642588919a8c548)) -* **metastore:** added support for release channels when creating service ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) -* **metastore:** Publish Dataproc Metastore v1alpha API ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) -* **metastore:** start generating apiv1alpha ([#3747](https://www.github.com/googleapis/google-cloud-go/issues/3747)) ([359312a](https://www.github.com/googleapis/google-cloud-go/commit/359312ad6d4f61fb341d41ffa35fc0634979e650)) -* **metastore:** start generating apiv1beta ([#3788](https://www.github.com/googleapis/google-cloud-go/issues/3788)) ([2977095](https://www.github.com/googleapis/google-cloud-go/commit/297709593ad32f234c0fbcfa228cffcfd3e591f4)) -* **secretmanager:** added topic field to Secret ([f1323b1](https://www.github.com/googleapis/google-cloud-go/commit/f1323b10a3c7cc1d215730cefd3062064ef54c01)) - - -### Bug Fixes - -* **analytics/admin:** add `https://www.googleapis.com/auth/analytics.edit` OAuth2 scope to the list of acceptable scopes for all read only methods of the Admin API docs: update the documentation of the `update_mask` field used by Update() methods ([f1323b1](https://www.github.com/googleapis/google-cloud-go/commit/f1323b10a3c7cc1d215730cefd3062064ef54c01)) -* **apigateway:** Provide resource definitions for service management and IAM resources ([18c88c4](https://www.github.com/googleapis/google-cloud-go/commit/18c88c437bd1741eaf5bf5911b9da6f6ea7cd75d)) -* **functions:** Fix service namespace in grpc_service_config. ([7811a34](https://www.github.com/googleapis/google-cloud-go/commit/7811a34ef64d722480c640810251bb3a0d65d495)) -* **internal/godocfx:** prevent index out of bounds when pkg == mod ([#3768](https://www.github.com/googleapis/google-cloud-go/issues/3768)) ([3d80b4e](https://www.github.com/googleapis/google-cloud-go/commit/3d80b4e93b0f7e857d6e9681d8d6a429750ecf80)) -* **internal/godocfx:** use correct anchor links ([#3738](https://www.github.com/googleapis/google-cloud-go/issues/3738)) ([919039a](https://www.github.com/googleapis/google-cloud-go/commit/919039a01a006c41e720218bd55f83ce98a5edef)) -* **internal:** fix Bash syntax ([#3779](https://www.github.com/googleapis/google-cloud-go/issues/3779)) ([3dd245d](https://www.github.com/googleapis/google-cloud-go/commit/3dd245dbdbfa84f0bbe5a476412d8463fe3e700c)) -* **tables:** use area120tables_v1alpha1.yaml as api-service-config ([#3759](https://www.github.com/googleapis/google-cloud-go/issues/3759)) ([b130ec0](https://www.github.com/googleapis/google-cloud-go/commit/b130ec0aa946b1a1eaa4d5a7c33e72353ac1612e)) - -## [0.78.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.77.0...v0.78.0) (2021-02-22) - - -### Features - -* **area120/tables:** Added ListWorkspaces, GetWorkspace, BatchDeleteRows APIs. ([16597fa](https://www.github.com/googleapis/google-cloud-go/commit/16597fa1ce549053c7183e8456e23f554a5501de)) -* **area120/tables:** Added ListWorkspaces, GetWorkspace, BatchDeleteRows APIs. ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) -* **dialogflow:** add additional_bindings to Dialogflow v2 ListIntents API docs: update copyrights and session docs ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) -* **documentai:** Update documentai/v1beta3 protos ([613ced7](https://www.github.com/googleapis/google-cloud-go/commit/613ced702bbc82a154a4d3641b483f71c7cd1af4)) -* **gkehub:** Update Membership API v1beta1 proto ([613ced7](https://www.github.com/googleapis/google-cloud-go/commit/613ced702bbc82a154a4d3641b483f71c7cd1af4)) -* **servicecontrol:** Update the ruby_cloud_gapic_library rules for the libraries published to google-cloud-ruby to the form that works with build_gen (separate parameters for ruby_cloud_title and ruby_cloud_description). chore: Update Bazel-Ruby rules version. chore: Update build_gen version. ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) -* **speech:** Support Model Adaptation. ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) - - -### Bug Fixes - -* **dialogflow/cx:** RunTestCase http template. PHP REST client lib can be generated. feat: Support transition route group coverage for Test Cases. ([613ced7](https://www.github.com/googleapis/google-cloud-go/commit/613ced702bbc82a154a4d3641b483f71c7cd1af4)) -* **errorreporting:** Fixes ruby gem build ([0bd21d7](https://www.github.com/googleapis/google-cloud-go/commit/0bd21d793f75924e5a2d033c58e8aaef89cf8113)) - -## [0.77.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.76.0...v0.77.0) (2021-02-16) - - -### Features - -* **channel:** Add Pub/Sub endpoints for Cloud Channel API. ([1aea7c8](https://www.github.com/googleapis/google-cloud-go/commit/1aea7c87d39eed87620b488ba0dd60b88ff26c04)) -* **dialogflow/cx:** supports SentimentAnalysisResult in webhook request docs: minor updates in wording ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) -* **errorreporting:** Make resolution status field available for error groups. Now callers can set the status of an error group by passing this to UpdateGroup. When not specified, it's treated like OPEN. feat: Make source location available for error groups created from GAE. ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) -* **errorreporting:** Make resolution status field available for error groups. Now callers can set the status of an error group by passing this to UpdateGroup. When not specified, it's treated like OPEN. feat: Make source location available for error groups created from GAE. ([f66114b](https://www.github.com/googleapis/google-cloud-go/commit/f66114bc7233ad06e18f38dd39497a74d85fdbd8)) -* **gkehub:** start generating apiv1beta1 ([#3698](https://www.github.com/googleapis/google-cloud-go/issues/3698)) ([8aed3bd](https://www.github.com/googleapis/google-cloud-go/commit/8aed3bd1bbbe983e4891c813e4c5dc9b3aa1b9b2)) -* **internal/docfx:** full cross reference linking ([#3656](https://www.github.com/googleapis/google-cloud-go/issues/3656)) ([fcb7318](https://www.github.com/googleapis/google-cloud-go/commit/fcb7318eb338bf3828ac831ed06ca630e1876418)) -* **memcache:** added ApplySoftwareUpdate API docs: various clarifications, new documentation for ApplySoftwareUpdate chore: update proto annotations ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) -* **networkconnectivity:** Add state field in resources docs: Minor changes ([0b4370a](https://www.github.com/googleapis/google-cloud-go/commit/0b4370a0d397913d932dbbdc2046a958dc3b836a)) -* **networkconnectivity:** Add state field in resources docs: Minor changes ([b4b5898](https://www.github.com/googleapis/google-cloud-go/commit/b4b58987368f80494bbc7f651f50e9123200fb3f)) -* **recommendationengine:** start generating apiv1beta1 ([#3686](https://www.github.com/googleapis/google-cloud-go/issues/3686)) ([8f4e130](https://www.github.com/googleapis/google-cloud-go/commit/8f4e13009444d88a5a56144129f055623a2205ac)) - - -### Bug Fixes - -* **errorreporting:** Remove dependency on AppEngine's proto definitions. This also removes the source_references field. ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) -* **errorreporting:** Update bazel builds for ER client libraries. ([0b4370a](https://www.github.com/googleapis/google-cloud-go/commit/0b4370a0d397913d932dbbdc2046a958dc3b836a)) -* **internal/godocfx:** use exact list of top-level decls ([#3665](https://www.github.com/googleapis/google-cloud-go/issues/3665)) ([3cd2961](https://www.github.com/googleapis/google-cloud-go/commit/3cd2961bd7b9c29d82a21ba8850eff00c7c332fd)) -* **kms:** do not retry on 13 INTERNAL ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) -* **orgpolicy:** Fix constraint resource pattern annotation ([f66114b](https://www.github.com/googleapis/google-cloud-go/commit/f66114bc7233ad06e18f38dd39497a74d85fdbd8)) -* **orgpolicy:** Fix constraint resource pattern annotation ([0b4370a](https://www.github.com/googleapis/google-cloud-go/commit/0b4370a0d397913d932dbbdc2046a958dc3b836a)) -* **profiler:** make sure retries use the most up-to-date copy of the trailer ([#3660](https://www.github.com/googleapis/google-cloud-go/issues/3660)) ([3ba9ebc](https://www.github.com/googleapis/google-cloud-go/commit/3ba9ebcee2b8b43cdf2c8f8a3d810516a604b363)) -* **vision:** sync vision v1 protos to get extra FaceAnnotation Landmark Types ([2b4414d](https://www.github.com/googleapis/google-cloud-go/commit/2b4414d973e3445725cd38901bf75340c97fc663)) - -## [0.76.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.75.0...v0.76.0) (2021-02-02) - - -### Features - -* **accessapproval:** Migrate the Bazel rules for the libraries published to google-cloud-ruby to use the gapic-generator-ruby instead of the monolith generator. ([ac22beb](https://www.github.com/googleapis/google-cloud-go/commit/ac22beb9b90771b24c8b35db7587ad3f5c0a970e)) -* **all:** auto-regenerate gapics ([#3526](https://www.github.com/googleapis/google-cloud-go/issues/3526)) ([ab2af0b](https://www.github.com/googleapis/google-cloud-go/commit/ab2af0b32630dd97f44800f4e273184f887375db)) -* **all:** auto-regenerate gapics ([#3539](https://www.github.com/googleapis/google-cloud-go/issues/3539)) ([84d4d8a](https://www.github.com/googleapis/google-cloud-go/commit/84d4d8ae2d3fbf34a4a312a0a2e4062d18caaa3d)) -* **all:** auto-regenerate gapics ([#3546](https://www.github.com/googleapis/google-cloud-go/issues/3546)) ([959fde5](https://www.github.com/googleapis/google-cloud-go/commit/959fde5ab12f7aee206dd46022e3cad1bc3470f7)) -* **all:** auto-regenerate gapics ([#3563](https://www.github.com/googleapis/google-cloud-go/issues/3563)) ([102112a](https://www.github.com/googleapis/google-cloud-go/commit/102112a4e9285a16645aabc89789f613d4f47c9e)) -* **all:** auto-regenerate gapics ([#3576](https://www.github.com/googleapis/google-cloud-go/issues/3576)) ([ac22beb](https://www.github.com/googleapis/google-cloud-go/commit/ac22beb9b90771b24c8b35db7587ad3f5c0a970e)) -* **all:** auto-regenerate gapics ([#3580](https://www.github.com/googleapis/google-cloud-go/issues/3580)) ([9974a80](https://www.github.com/googleapis/google-cloud-go/commit/9974a8017b5de8129a586f2404a23396caea0ee1)) -* **all:** auto-regenerate gapics ([#3587](https://www.github.com/googleapis/google-cloud-go/issues/3587)) ([3859a6f](https://www.github.com/googleapis/google-cloud-go/commit/3859a6ffc447e9c0b4ef231e2788fbbcfe48a94f)) -* **all:** auto-regenerate gapics ([#3598](https://www.github.com/googleapis/google-cloud-go/issues/3598)) ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **appengine:** start generating apiv1 ([#3561](https://www.github.com/googleapis/google-cloud-go/issues/3561)) ([2b6a3b4](https://www.github.com/googleapis/google-cloud-go/commit/2b6a3b4609e389da418a83eb60a8ae3710d646d7)) -* **assuredworkloads:** updated google.cloud.assuredworkloads.v1beta1.AssuredWorkloadsService service. Clients can now create workloads with US_REGIONAL_ACCESS compliance regime ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **binaryauthorization:** start generating apiv1beta1 ([#3562](https://www.github.com/googleapis/google-cloud-go/issues/3562)) ([56e18a6](https://www.github.com/googleapis/google-cloud-go/commit/56e18a64836ab9482528b212eb139f649f7a35c3)) -* **channel:** Add Pub/Sub endpoints for Cloud Channel API. ([9070c86](https://www.github.com/googleapis/google-cloud-go/commit/9070c86e2c69f9405d42fc0e6fe7afd4a256d8b8)) -* **cloudtasks:** introducing field: ListQueuesRequest.read_mask, GetQueueRequest.read_mask, Queue.task_ttl, Queue.tombstone_ttl, Queue.stats, Task.pull_message and introducing messages: QueueStats PullMessage docs: updates to max burst size description ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **cloudtasks:** introducing fields: ListQueuesRequest.read_mask, GetQueueRequest.read_mask, Queue.task_ttl, Queue.tombstone_ttl, Queue.stats and introducing messages: QueueStats docs: updates to AppEngineHttpRequest description ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **datalabeling:** start generating apiv1beta1 ([#3582](https://www.github.com/googleapis/google-cloud-go/issues/3582)) ([d8a7fee](https://www.github.com/googleapis/google-cloud-go/commit/d8a7feef51d3344fa7e258aba1d9fbdab56dadcf)) -* **dataqna:** start generating apiv1alpha ([#3586](https://www.github.com/googleapis/google-cloud-go/issues/3586)) ([24c5b8f](https://www.github.com/googleapis/google-cloud-go/commit/24c5b8f4f45f8cd8b3001b1ca5a8d80e9f3b39d5)) -* **dialogflow/cx:** Add new Experiment service docs: minor doc update on redact field in intent.proto and page.proto ([0959f27](https://www.github.com/googleapis/google-cloud-go/commit/0959f27e85efe94d39437ceef0ff62ddceb8e7a7)) -* **dialogflow/cx:** added support for test cases and agent validation ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **dialogflow/cx:** added support for test cases and agent validation ([3859a6f](https://www.github.com/googleapis/google-cloud-go/commit/3859a6ffc447e9c0b4ef231e2788fbbcfe48a94f)) -* **dialogflow:** add C++ targets for DialogFlow ([959fde5](https://www.github.com/googleapis/google-cloud-go/commit/959fde5ab12f7aee206dd46022e3cad1bc3470f7)) -* **documentai:** start generating apiv1beta3 ([#3595](https://www.github.com/googleapis/google-cloud-go/issues/3595)) ([5ae21fa](https://www.github.com/googleapis/google-cloud-go/commit/5ae21fa1cfb8b8dacbcd0fc43eee430f7db63102)) -* **domains:** start generating apiv1beta1 ([#3632](https://www.github.com/googleapis/google-cloud-go/issues/3632)) ([b8ada6f](https://www.github.com/googleapis/google-cloud-go/commit/b8ada6f197e680d0bb26aa031e6431bc099a3149)) -* **godocfx:** include alt documentation link ([#3530](https://www.github.com/googleapis/google-cloud-go/issues/3530)) ([806cdd5](https://www.github.com/googleapis/google-cloud-go/commit/806cdd56fb6fdddd7a6c1354e55e0d1259bd6c8b)) -* **internal/gapicgen:** change commit formatting to match standard ([#3500](https://www.github.com/googleapis/google-cloud-go/issues/3500)) ([d1e3d46](https://www.github.com/googleapis/google-cloud-go/commit/d1e3d46c47c425581e2b149c07f8e27ffc373c7e)) -* **internal/godocfx:** xref function declarations ([#3615](https://www.github.com/googleapis/google-cloud-go/issues/3615)) ([2bdbb87](https://www.github.com/googleapis/google-cloud-go/commit/2bdbb87a682d799cf5e262a61a3ef1faf41151af)) -* **mediatranslation:** start generating apiv1beta1 ([#3636](https://www.github.com/googleapis/google-cloud-go/issues/3636)) ([4129469](https://www.github.com/googleapis/google-cloud-go/commit/412946966cf7f53c51deff1b1cc1a12d62ed0279)) -* **memcache:** start generating apiv1 ([#3579](https://www.github.com/googleapis/google-cloud-go/issues/3579)) ([eabf7cf](https://www.github.com/googleapis/google-cloud-go/commit/eabf7cfde7b3a3cc1b35c320ba52e07be9926359)) -* **networkconnectivity:** initial generation of apiv1alpha1 ([#3567](https://www.github.com/googleapis/google-cloud-go/issues/3567)) ([adf489a](https://www.github.com/googleapis/google-cloud-go/commit/adf489a536292e3196677621477eae0d52761e7f)) -* **orgpolicy:** start generating apiv2 ([#3652](https://www.github.com/googleapis/google-cloud-go/issues/3652)) ([c103847](https://www.github.com/googleapis/google-cloud-go/commit/c1038475779fda3589aa9659d4ad0b703036b531)) -* **osconfig/agentendpoint:** add ApplyConfigTask to AgentEndpoint API ([9070c86](https://www.github.com/googleapis/google-cloud-go/commit/9070c86e2c69f9405d42fc0e6fe7afd4a256d8b8)) -* **osconfig/agentendpoint:** add ApplyConfigTask to AgentEndpoint API ([9af529c](https://www.github.com/googleapis/google-cloud-go/commit/9af529c21e98b62c4617f7a7191c307659cf8bb8)) -* **recommender:** add bindings for folder/org type resources for protos in recommendations, insights and recommender_service to enable v1 api for folder/org ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **recommender:** auto generated cl for enabling v1beta1 folder/org APIs and integration test ([7bdebad](https://www.github.com/googleapis/google-cloud-go/commit/7bdebadbe06774c94ab745dfef4ce58ce40a5582)) -* **resourcemanager:** start generating apiv2 ([#3575](https://www.github.com/googleapis/google-cloud-go/issues/3575)) ([93d0ebc](https://www.github.com/googleapis/google-cloud-go/commit/93d0ebceb4270351518a13958005bb68f0cace60)) -* **secretmanager:** added expire_time and ttl fields to Secret ([9974a80](https://www.github.com/googleapis/google-cloud-go/commit/9974a8017b5de8129a586f2404a23396caea0ee1)) -* **secretmanager:** added expire_time and ttl fields to Secret ([ac22beb](https://www.github.com/googleapis/google-cloud-go/commit/ac22beb9b90771b24c8b35db7587ad3f5c0a970e)) -* **servicecontrol:** start generating apiv1 ([#3644](https://www.github.com/googleapis/google-cloud-go/issues/3644)) ([f84938b](https://www.github.com/googleapis/google-cloud-go/commit/f84938bb4042a5629fd66bda42de028fd833648a)) -* **servicemanagement:** start generating apiv1 ([#3614](https://www.github.com/googleapis/google-cloud-go/issues/3614)) ([b96134f](https://www.github.com/googleapis/google-cloud-go/commit/b96134fe91c182237359000cd544af5fec60d7db)) - - -### Bug Fixes - -* **datacatalog:** Update PHP package name casing to match the PHP namespace in the proto files ([c7ecf0f](https://www.github.com/googleapis/google-cloud-go/commit/c7ecf0f3f454606b124e52d20af2545b2c68646f)) -* **internal/godocfx:** add TOC element for module root package ([#3599](https://www.github.com/googleapis/google-cloud-go/issues/3599)) ([1d6eb23](https://www.github.com/googleapis/google-cloud-go/commit/1d6eb238206fcf8815d88981527ef176851afd7a)) -* **profiler:** Force gax to retry in case of certificate errors ([#3178](https://www.github.com/googleapis/google-cloud-go/issues/3178)) ([35dcd72](https://www.github.com/googleapis/google-cloud-go/commit/35dcd725dcd03266ed7439de40c277376b38cd71)) - -## [0.75.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.74.0...v0.75.0) (2021-01-11) - - -### Features - -* **all:** auto-regenerate gapics , refs [#3514](https://www.github.com/googleapis/google-cloud-go/issues/3514) [#3501](https://www.github.com/googleapis/google-cloud-go/issues/3501) [#3497](https://www.github.com/googleapis/google-cloud-go/issues/3497) [#3455](https://www.github.com/googleapis/google-cloud-go/issues/3455) [#3448](https://www.github.com/googleapis/google-cloud-go/issues/3448) -* **channel:** start generating apiv1 ([#3517](https://www.github.com/googleapis/google-cloud-go/issues/3517)) ([2cf3b3c](https://www.github.com/googleapis/google-cloud-go/commit/2cf3b3cf7d99f2efd6868a710fad9e935fc87965)) - - -### Bug Fixes - -* **internal/gapicgen:** don't regen files that have been deleted ([#3471](https://www.github.com/googleapis/google-cloud-go/issues/3471)) ([112ca94](https://www.github.com/googleapis/google-cloud-go/commit/112ca9416cc8a2502b32547dc8d789655452f84a)) - -## [0.74.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.73.0...v0.74.0) (2020-12-10) - - -### Features - -* **all:** auto-regenerate gapics , refs [#3440](https://www.github.com/googleapis/google-cloud-go/issues/3440) [#3436](https://www.github.com/googleapis/google-cloud-go/issues/3436) [#3394](https://www.github.com/googleapis/google-cloud-go/issues/3394) [#3391](https://www.github.com/googleapis/google-cloud-go/issues/3391) [#3374](https://www.github.com/googleapis/google-cloud-go/issues/3374) -* **internal/gapicgen:** support generating only gapics with genlocal ([#3383](https://www.github.com/googleapis/google-cloud-go/issues/3383)) ([eaa742a](https://www.github.com/googleapis/google-cloud-go/commit/eaa742a248dc7d93c019863248f28e37f88aae84)) -* **servicedirectory:** start generating apiv1 ([#3382](https://www.github.com/googleapis/google-cloud-go/issues/3382)) ([2774925](https://www.github.com/googleapis/google-cloud-go/commit/2774925925909071ebc585cf7400373334c156ba)) - - -### Bug Fixes - -* **internal/gapicgen:** don't create genproto pr as draft ([#3379](https://www.github.com/googleapis/google-cloud-go/issues/3379)) ([517ab0f](https://www.github.com/googleapis/google-cloud-go/commit/517ab0f25e544498c5374b256354bc41ba936ad5)) - -## [0.73.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.72.0...v0.73.0) (2020-12-04) - - -### Features - -* **all:** auto-regenerate gapics , refs [#3335](https://www.github.com/googleapis/google-cloud-go/issues/3335) [#3294](https://www.github.com/googleapis/google-cloud-go/issues/3294) [#3250](https://www.github.com/googleapis/google-cloud-go/issues/3250) [#3229](https://www.github.com/googleapis/google-cloud-go/issues/3229) [#3211](https://www.github.com/googleapis/google-cloud-go/issues/3211) [#3217](https://www.github.com/googleapis/google-cloud-go/issues/3217) [#3212](https://www.github.com/googleapis/google-cloud-go/issues/3212) [#3209](https://www.github.com/googleapis/google-cloud-go/issues/3209) [#3206](https://www.github.com/googleapis/google-cloud-go/issues/3206) [#3199](https://www.github.com/googleapis/google-cloud-go/issues/3199) -* **artifactregistry:** start generating apiv1beta2 ([#3352](https://www.github.com/googleapis/google-cloud-go/issues/3352)) ([2e6f20b](https://www.github.com/googleapis/google-cloud-go/commit/2e6f20b0ab438b0b366a1a3802fc64d1a0e66fff)) -* **internal:** copy pubsub Message and PublishResult to internal/pubsub ([#3351](https://www.github.com/googleapis/google-cloud-go/issues/3351)) ([82521ee](https://www.github.com/googleapis/google-cloud-go/commit/82521ee5038735c1663525658d27e4df00ec90be)) -* **internal/gapicgen:** support adding context to regen ([#3174](https://www.github.com/googleapis/google-cloud-go/issues/3174)) ([941ab02](https://www.github.com/googleapis/google-cloud-go/commit/941ab029ba6f7f33e8b2e31e3818aeb68312a999)) -* **internal/kokoro:** add ability to regen all DocFX YAML ([#3191](https://www.github.com/googleapis/google-cloud-go/issues/3191)) ([e12046b](https://www.github.com/googleapis/google-cloud-go/commit/e12046bc4431d33aee72c324e6eb5cc907a4214a)) - - -### Bug Fixes - -* **internal/godocfx:** filter out test packages from other modules ([#3197](https://www.github.com/googleapis/google-cloud-go/issues/3197)) ([1d397aa](https://www.github.com/googleapis/google-cloud-go/commit/1d397aa8b41f8f980cba1d3dcc50f11e4d4f4ca0)) - -## [0.72.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.71.0...v0.72.0) (2020-11-10) - - -### Features - -* **all:** auto-regenerate gapics , refs [#3177](https://www.github.com/googleapis/google-cloud-go/issues/3177) [#3164](https://www.github.com/googleapis/google-cloud-go/issues/3164) [#3149](https://www.github.com/googleapis/google-cloud-go/issues/3149) [#3142](https://www.github.com/googleapis/google-cloud-go/issues/3142) [#3136](https://www.github.com/googleapis/google-cloud-go/issues/3136) [#3130](https://www.github.com/googleapis/google-cloud-go/issues/3130) [#3121](https://www.github.com/googleapis/google-cloud-go/issues/3121) [#3119](https://www.github.com/googleapis/google-cloud-go/issues/3119) - - -### Bug Fixes - -* **all:** Update hand-written clients to not use WithEndpoint override ([#3111](https://www.github.com/googleapis/google-cloud-go/issues/3111)) ([f0cfd05](https://www.github.com/googleapis/google-cloud-go/commit/f0cfd0532f5204ff16f7bae406efa72603d16f44)) -* **internal/godocfx:** rename README files to pkg-readme ([#3185](https://www.github.com/googleapis/google-cloud-go/issues/3185)) ([d3a8571](https://www.github.com/googleapis/google-cloud-go/commit/d3a85719be411b692aede3331abb29b5a7b3da9a)) - - -## [0.71.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.70.0...v0.71.0) (2020-10-30) - - -### Features - -* **all:** auto-regenerate gapics , refs [#3115](https://www.github.com/googleapis/google-cloud-go/issues/3115) [#3106](https://www.github.com/googleapis/google-cloud-go/issues/3106) [#3102](https://www.github.com/googleapis/google-cloud-go/issues/3102) [#3083](https://www.github.com/googleapis/google-cloud-go/issues/3083) [#3073](https://www.github.com/googleapis/google-cloud-go/issues/3073) [#3057](https://www.github.com/googleapis/google-cloud-go/issues/3057) [#3044](https://www.github.com/googleapis/google-cloud-go/issues/3044) -* **billing/budgets:** start generating apiv1 ([#3099](https://www.github.com/googleapis/google-cloud-go/issues/3099)) ([e760c85](https://www.github.com/googleapis/google-cloud-go/commit/e760c859de88a6e79b6dffc653dbf75f1630d8e3)) -* **internal:** auto-run godocfx on new mods ([#3069](https://www.github.com/googleapis/google-cloud-go/issues/3069)) ([49f497e](https://www.github.com/googleapis/google-cloud-go/commit/49f497eab80ce34dfb4ca41f033a5c0429ff5e42)) -* **pubsublite:** Added Pub/Sub Lite clients and routing headers ([#3105](https://www.github.com/googleapis/google-cloud-go/issues/3105)) ([98668fa](https://www.github.com/googleapis/google-cloud-go/commit/98668fa5457d26ed34debee708614f027020e5bc)) -* **pubsublite:** Message type and message routers ([#3077](https://www.github.com/googleapis/google-cloud-go/issues/3077)) ([179fc55](https://www.github.com/googleapis/google-cloud-go/commit/179fc550b545a5344358a243da7007ffaa7b5171)) -* **pubsublite:** Pub/Sub Lite admin client ([#3036](https://www.github.com/googleapis/google-cloud-go/issues/3036)) ([749473e](https://www.github.com/googleapis/google-cloud-go/commit/749473ead30bf1872634821d3238d1299b99acc6)) -* **pubsublite:** Publish settings and errors ([#3075](https://www.github.com/googleapis/google-cloud-go/issues/3075)) ([9eb9fcb](https://www.github.com/googleapis/google-cloud-go/commit/9eb9fcb79f17ad7c08c77c455ba3e8d89e3bdbf2)) -* **pubsublite:** Retryable stream wrapper ([#3068](https://www.github.com/googleapis/google-cloud-go/issues/3068)) ([97cfd45](https://www.github.com/googleapis/google-cloud-go/commit/97cfd4587f2f51996bd685ff486308b70eb51900)) - - -### Bug Fixes - -* **internal/kokoro:** remove unnecessary cd ([#3071](https://www.github.com/googleapis/google-cloud-go/issues/3071)) ([c1a4c3e](https://www.github.com/googleapis/google-cloud-go/commit/c1a4c3eaffcdc3cffe0e223fcfa1f60879cd23bb)) -* **pubsublite:** Disable integration tests for project id ([#3087](https://www.github.com/googleapis/google-cloud-go/issues/3087)) ([a0982f7](https://www.github.com/googleapis/google-cloud-go/commit/a0982f79d6461feabdf31363f29fed7dc5677fe7)) - -## [0.70.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.69.0...v0.70.0) (2020-10-19) - - -### Features - -* **all:** auto-regenerate gapics , refs [#3047](https://www.github.com/googleapis/google-cloud-go/issues/3047) [#3035](https://www.github.com/googleapis/google-cloud-go/issues/3035) [#3025](https://www.github.com/googleapis/google-cloud-go/issues/3025) -* **managedidentities:** start generating apiv1 ([#3032](https://www.github.com/googleapis/google-cloud-go/issues/3032)) ([10ccca2](https://www.github.com/googleapis/google-cloud-go/commit/10ccca238074d24fea580a4cd8e64478818b0b44)) -* **pubsublite:** Types for resource paths and topic/subscription configs ([#3026](https://www.github.com/googleapis/google-cloud-go/issues/3026)) ([6f7fa86](https://www.github.com/googleapis/google-cloud-go/commit/6f7fa86ed906258f98d996aab40184f3a46f9714)) - -## [0.69.1](https://www.github.com/googleapis/google-cloud-go/compare/v0.69.0...v0.69.1) (2020-10-14) - -This is an empty release that was created solely to aid in pubsublite's module -carve out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## [0.69.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.68.0...v0.69.0) (2020-10-14) - - -### Features - -* **accessapproval:** start generating apiv1 ([#3002](https://www.github.com/googleapis/google-cloud-go/issues/3002)) ([709d6e7](https://www.github.com/googleapis/google-cloud-go/commit/709d6e76393e6ac00ff488efd83bfe873173b045)) -* **all:** auto-regenerate gapics , refs [#3010](https://www.github.com/googleapis/google-cloud-go/issues/3010) [#3005](https://www.github.com/googleapis/google-cloud-go/issues/3005) [#2993](https://www.github.com/googleapis/google-cloud-go/issues/2993) [#2989](https://www.github.com/googleapis/google-cloud-go/issues/2989) [#2981](https://www.github.com/googleapis/google-cloud-go/issues/2981) [#2976](https://www.github.com/googleapis/google-cloud-go/issues/2976) [#2968](https://www.github.com/googleapis/google-cloud-go/issues/2968) [#2958](https://www.github.com/googleapis/google-cloud-go/issues/2958) -* **cmd/go-cloud-debug-agent:** mark as deprecated ([#2964](https://www.github.com/googleapis/google-cloud-go/issues/2964)) ([276ec88](https://www.github.com/googleapis/google-cloud-go/commit/276ec88b05852c33a3ba437e18d072f7ffd8fd33)) -* **godocfx:** add nesting to TOC ([#2972](https://www.github.com/googleapis/google-cloud-go/issues/2972)) ([3a49b2d](https://www.github.com/googleapis/google-cloud-go/commit/3a49b2d142a353f98429235c3f380431430b4dbf)) -* **internal/godocfx:** HTML-ify package summary ([#2986](https://www.github.com/googleapis/google-cloud-go/issues/2986)) ([9e64b01](https://www.github.com/googleapis/google-cloud-go/commit/9e64b018255bd8d9b31d60e8f396966251de946b)) -* **internal/kokoro:** make publish_docs VERSION optional ([#2979](https://www.github.com/googleapis/google-cloud-go/issues/2979)) ([76e35f6](https://www.github.com/googleapis/google-cloud-go/commit/76e35f689cb60bd5db8e14b8c8d367c5902bcb0e)) -* **websecurityscanner:** start generating apiv1 ([#3006](https://www.github.com/googleapis/google-cloud-go/issues/3006)) ([1d92e20](https://www.github.com/googleapis/google-cloud-go/commit/1d92e2062a13f62d7a96be53a7354c0cacca6a85)) - - -### Bug Fixes - -* **godocfx:** make extra files optional, filter out third_party ([#2985](https://www.github.com/googleapis/google-cloud-go/issues/2985)) ([f268921](https://www.github.com/googleapis/google-cloud-go/commit/f2689214a24b2e325d3e8f54441bb11fbef925f0)) - -## [0.68.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.67.0...v0.68.0) (2020-10-02) - - -### Features - -* **all:** auto-regenerate gapics , refs [#2952](https://www.github.com/googleapis/google-cloud-go/issues/2952) [#2944](https://www.github.com/googleapis/google-cloud-go/issues/2944) [#2935](https://www.github.com/googleapis/google-cloud-go/issues/2935) - -## [0.67.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.66.0...v0.67.0) (2020-09-29) - - -### Features - -* **all:** auto-regenerate gapics , refs [#2933](https://www.github.com/googleapis/google-cloud-go/issues/2933) [#2919](https://www.github.com/googleapis/google-cloud-go/issues/2919) [#2913](https://www.github.com/googleapis/google-cloud-go/issues/2913) [#2910](https://www.github.com/googleapis/google-cloud-go/issues/2910) [#2899](https://www.github.com/googleapis/google-cloud-go/issues/2899) [#2897](https://www.github.com/googleapis/google-cloud-go/issues/2897) [#2886](https://www.github.com/googleapis/google-cloud-go/issues/2886) [#2877](https://www.github.com/googleapis/google-cloud-go/issues/2877) [#2869](https://www.github.com/googleapis/google-cloud-go/issues/2869) [#2864](https://www.github.com/googleapis/google-cloud-go/issues/2864) -* **assuredworkloads:** start generating apiv1beta1 ([#2866](https://www.github.com/googleapis/google-cloud-go/issues/2866)) ([7598c4d](https://www.github.com/googleapis/google-cloud-go/commit/7598c4dd2462e8270a2c7b1f496af58ca81ff568)) -* **dialogflow/cx:** start generating apiv3beta1 ([#2875](https://www.github.com/googleapis/google-cloud-go/issues/2875)) ([37ca93a](https://www.github.com/googleapis/google-cloud-go/commit/37ca93ad69eda363d956f0174d444ed5914f5a72)) -* **docfx:** add support for examples ([#2884](https://www.github.com/googleapis/google-cloud-go/issues/2884)) ([0cc0de3](https://www.github.com/googleapis/google-cloud-go/commit/0cc0de300d58be6d3b7eeb2f1baebfa6df076830)) -* **godocfx:** include README in output ([#2927](https://www.github.com/googleapis/google-cloud-go/issues/2927)) ([f084690](https://www.github.com/googleapis/google-cloud-go/commit/f084690a2ea08ce73bafaaced95ad271fd01e11e)) -* **talent:** start generating apiv4 ([#2871](https://www.github.com/googleapis/google-cloud-go/issues/2871)) ([5c98071](https://www.github.com/googleapis/google-cloud-go/commit/5c98071b03822c58862d1fa5442ff36d627f1a61)) - - -### Bug Fixes - -* **godocfx:** filter out other modules, sort pkgs ([#2894](https://www.github.com/googleapis/google-cloud-go/issues/2894)) ([868db45](https://www.github.com/googleapis/google-cloud-go/commit/868db45e2e6f4e9ad48432be86c849f335e1083d)) -* **godocfx:** shorten function names ([#2880](https://www.github.com/googleapis/google-cloud-go/issues/2880)) ([48a0217](https://www.github.com/googleapis/google-cloud-go/commit/48a0217930750c1f4327f2622b0f2a3ec8afc0b7)) -* **translate:** properly name examples ([#2892](https://www.github.com/googleapis/google-cloud-go/issues/2892)) ([c19e141](https://www.github.com/googleapis/google-cloud-go/commit/c19e1415e6fa76b7ea66a7fc67ad3ba22670a2ba)), refs [#2883](https://www.github.com/googleapis/google-cloud-go/issues/2883) - -## [0.66.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.65.0...v0.66.0) (2020-09-15) - - -### Features - -* **all:** auto-regenerate gapics , refs [#2849](https://www.github.com/googleapis/google-cloud-go/issues/2849) [#2843](https://www.github.com/googleapis/google-cloud-go/issues/2843) [#2841](https://www.github.com/googleapis/google-cloud-go/issues/2841) [#2819](https://www.github.com/googleapis/google-cloud-go/issues/2819) [#2816](https://www.github.com/googleapis/google-cloud-go/issues/2816) [#2809](https://www.github.com/googleapis/google-cloud-go/issues/2809) [#2801](https://www.github.com/googleapis/google-cloud-go/issues/2801) [#2795](https://www.github.com/googleapis/google-cloud-go/issues/2795) [#2791](https://www.github.com/googleapis/google-cloud-go/issues/2791) [#2788](https://www.github.com/googleapis/google-cloud-go/issues/2788) [#2781](https://www.github.com/googleapis/google-cloud-go/issues/2781) -* **analytics/data:** start generating apiv1alpha ([#2796](https://www.github.com/googleapis/google-cloud-go/issues/2796)) ([e93132c](https://www.github.com/googleapis/google-cloud-go/commit/e93132c77725de3c80c34d566df269eabfcfde93)) -* **area120/tables:** start generating apiv1alpha1 ([#2807](https://www.github.com/googleapis/google-cloud-go/issues/2807)) ([9e5a4d0](https://www.github.com/googleapis/google-cloud-go/commit/9e5a4d0dee0d83be0c020797a2f579d9e42ef521)) -* **cloudbuild:** Start generating apiv1/v3 ([#2830](https://www.github.com/googleapis/google-cloud-go/issues/2830)) ([358a536](https://www.github.com/googleapis/google-cloud-go/commit/358a5368da64cf4868551652e852ceb453504f64)) -* **godocfx:** create Go DocFX YAML generator ([#2854](https://www.github.com/googleapis/google-cloud-go/issues/2854)) ([37c70ac](https://www.github.com/googleapis/google-cloud-go/commit/37c70acd91768567106ff3b2b130835998d974c5)) -* **security/privateca:** start generating apiv1beta1 ([#2806](https://www.github.com/googleapis/google-cloud-go/issues/2806)) ([f985141](https://www.github.com/googleapis/google-cloud-go/commit/f9851412183989dc69733a7e61ad39a9378cd893)) -* **video/transcoder:** start generating apiv1beta1 ([#2797](https://www.github.com/googleapis/google-cloud-go/issues/2797)) ([390dda8](https://www.github.com/googleapis/google-cloud-go/commit/390dda8ff2c526e325e434ad0aec778b7aa97ea4)) -* **workflows:** start generating apiv1beta ([#2799](https://www.github.com/googleapis/google-cloud-go/issues/2799)) ([0e39665](https://www.github.com/googleapis/google-cloud-go/commit/0e39665ccb788caec800e2887d433ca6e0cf9901)) -* **workflows/executions:** start generating apiv1beta ([#2800](https://www.github.com/googleapis/google-cloud-go/issues/2800)) ([7eaa0d1](https://www.github.com/googleapis/google-cloud-go/commit/7eaa0d184c6a2141d8bf4514b3fd20715b50a580)) - - -### Bug Fixes - -* **internal/kokoro:** install the right version of docuploader ([#2861](https://www.github.com/googleapis/google-cloud-go/issues/2861)) ([d8489c1](https://www.github.com/googleapis/google-cloud-go/commit/d8489c141b8b02e83d6426f4baebd3658ae11639)) -* **internal/kokoro:** remove extra dash in doc tarball ([#2862](https://www.github.com/googleapis/google-cloud-go/issues/2862)) ([690ddcc](https://www.github.com/googleapis/google-cloud-go/commit/690ddccc5202b5a70f1afa5c518dca37b6a0861c)) -* **profiler:** do not collect disabled profile types ([#2836](https://www.github.com/googleapis/google-cloud-go/issues/2836)) ([faeb498](https://www.github.com/googleapis/google-cloud-go/commit/faeb4985bf6afdcddba4553efa874642bf7f08ed)), refs [#2835](https://www.github.com/googleapis/google-cloud-go/issues/2835) - - -### Reverts - -* **cloudbuild): "feat(cloudbuild:** Start generating apiv1/v3" ([#2840](https://www.github.com/googleapis/google-cloud-go/issues/2840)) ([3aaf755](https://www.github.com/googleapis/google-cloud-go/commit/3aaf755476dfea1700986fc086f53fc1ab756557)) - -## [0.65.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.64.0...v0.65.0) (2020-08-27) - - -### Announcements - -The following changes will be included in an upcoming release and are not -included in this one. - -#### Default Deadlines - -By default, non-streaming methods, like Create or Get methods, will have a -default deadline applied to the context provided at call time, unless a context -deadline is already set. Streaming methods have no default deadline and will run -indefinitely, unless the context provided at call time contains a deadline. - -To opt-out of this behavior, set the environment variable -`GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE` to `true` prior to -initializing a client. This opt-out mechanism will be removed in a later -release, with a notice similar to this one ahead of its removal. - - -### Features - -* **all:** auto-regenerate gapics , refs [#2774](https://www.github.com/googleapis/google-cloud-go/issues/2774) [#2764](https://www.github.com/googleapis/google-cloud-go/issues/2764) - - -### Bug Fixes - -* **all:** correct minor typos ([#2756](https://www.github.com/googleapis/google-cloud-go/issues/2756)) ([03d78b5](https://www.github.com/googleapis/google-cloud-go/commit/03d78b5627819cb64d1f3866f90043f709e825e1)) -* **compute/metadata:** remove leading slash for Get suffix ([#2760](https://www.github.com/googleapis/google-cloud-go/issues/2760)) ([f0d605c](https://www.github.com/googleapis/google-cloud-go/commit/f0d605ccf32391a9da056a2c551158bd076c128d)) - -## [0.64.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.63.0...v0.64.0) (2020-08-18) - - -### Features - -* **all:** auto-regenerate gapics , refs [#2734](https://www.github.com/googleapis/google-cloud-go/issues/2734) [#2731](https://www.github.com/googleapis/google-cloud-go/issues/2731) [#2730](https://www.github.com/googleapis/google-cloud-go/issues/2730) [#2725](https://www.github.com/googleapis/google-cloud-go/issues/2725) [#2722](https://www.github.com/googleapis/google-cloud-go/issues/2722) [#2706](https://www.github.com/googleapis/google-cloud-go/issues/2706) -* **pubsublite:** start generating v1 ([#2700](https://www.github.com/googleapis/google-cloud-go/issues/2700)) ([d2e777f](https://www.github.com/googleapis/google-cloud-go/commit/d2e777f56e08146646b3ffb7a78856795094ab4e)) - -## [0.63.0](https://www.github.com/googleapis/google-cloud-go/compare/v0.62.0...v0.63.0) (2020-08-05) - - -### Features - -* **all:** auto-regenerate gapics ([#2682](https://www.github.com/googleapis/google-cloud-go/issues/2682)) ([63bfd63](https://www.github.com/googleapis/google-cloud-go/commit/63bfd638da169e0f1f4fa4a5125da2955022dc04)) -* **analytics/admin:** start generating apiv1alpha ([#2670](https://www.github.com/googleapis/google-cloud-go/issues/2670)) ([268199e](https://www.github.com/googleapis/google-cloud-go/commit/268199e5350a64a83ecf198e0e0fa4863f00fa6c)) -* **functions/metadata:** Special-case marshaling ([#2669](https://www.github.com/googleapis/google-cloud-go/issues/2669)) ([d8d7fc6](https://www.github.com/googleapis/google-cloud-go/commit/d8d7fc66cbc42f79bec25fb0daaf53d926e3645b)) -* **gaming:** start generate apiv1 ([#2681](https://www.github.com/googleapis/google-cloud-go/issues/2681)) ([1adfd0a](https://www.github.com/googleapis/google-cloud-go/commit/1adfd0aed6b2c0e1dd0c575a5ec0f49388fa5601)) -* **internal/kokoro:** add script to test compatibility with samples ([#2637](https://www.github.com/googleapis/google-cloud-go/issues/2637)) ([f2aa76a](https://www.github.com/googleapis/google-cloud-go/commit/f2aa76a0058e86c1c33bb634d2c084b58f77ab32)) - -## v0.62.0 - -### Announcements - -- There was a breaking change to `cloud.google.com/go/dataproc/apiv1` that was - merged in [this PR](https://github.com/googleapis/google-cloud-go/pull/2606). - This fixed a broken API response for `DiagnoseCluster`. When polling on the - Long Running Operation(LRO), the API now returns - `(*dataprocpb.DiagnoseClusterResults, error)` whereas it only returned an - `error` before. - -### Changes - -- all: - - Updated all direct dependencies. - - Updated contributing guidelines to suggest allowing edits from maintainers. -- billing/budgets: - - Start generating client for apiv1beta1. -- functions: - - Start generating client for apiv1. -- notebooks: - - Start generating client apiv1beta1. -- profiler: - - update proftest to support parsing floating-point backoff durations. - - Fix the regexp used to parse backoff duration. -- Various updates to autogenerated clients. - -## v0.61.0 - -### Changes - -- all: - - Update all direct dependencies. -- dashboard: - - Start generating client for apiv1. -- policytroubleshooter: - - Start generating client for apiv1. -- profiler: - - Disable OpenCensus Telemetry for requests made by the profiler package by default. You can re-enable it using `profiler.Config.EnableOCTelemetry`. -- Various updates to autogenerated clients. - -## v0.60.0 - -### Changes - -- all: - - Refactored examples to reduce module dependencies. - - Update sub-modules to use cloud.google.com/go v0.59.0. -- internal: - - Start generating client for gaming apiv1beta. -- Various updates to autogenerated clients. - -## v0.59.0 - -### Announcements - -goolgeapis/google-cloud-go has moved its source of truth to GitHub and is no longer a mirror. This means that our -contributing process has changed a bit. We will now be conducting all code reviews on GitHub which means we now accept -pull requests! If you have a version of the codebase previously checked out you may wish to update your git remote to -point to GitHub. - -### Changes - -- all: - - Remove dependency on honnef.co/go/tools. - - Update our contributing instructions now that we use GitHub for reviews. - - Remove some un-inclusive terminology. -- compute/metadata: - - Pass cancelable context to DNS lookup. -- .github: - - Update templates issue/PR templates. -- internal: - - Bump several clients to GA. - - Fix GoDoc badge source. - - Several automation changes related to the move to GitHub. - - Start generating a client for asset v1p5beta1. -- Various updates to autogenerated clients. - -## v0.58.0 - -### Deprecation notice - -- `cloud.google.com/go/monitoring/apiv3` has been deprecated due to breaking - changes in the API. Please migrate to `cloud.google.com/go/monitoring/apiv3/v2`. - -### Changes - -- all: - - The remaining uses of gtransport.Dial have been removed. - - The `genproto` dependency has been updated to a version that makes use of - new `protoreflect` library. For more information on these protobuf changes - please see the following post from the official Go blog: - https://blog.golang.org/protobuf-apiv2. -- internal: - - Started generation of datastore admin v1 client. - - Updated protofuf version used for generation to 3.12.X. - - Update the release levels for several APIs. - - Generate clients with protoc-gen-go@v1.4.1. -- monitoring: - - Re-enable generation of monitoring/apiv3 under v2 directory (see deprecation - notice above). -- profiler: - - Fixed flakiness in tests. -- Various updates to autogenerated clients. - -## v0.57.0 - -- all: - - Update module dependency `google.golang.org/api` to `v0.21.0`. -- errorreporting: - - Add exported SetGoogleClientInfo wrappers to manual file. -- expr/v1alpha1: - - Deprecate client. This client will be removed in a future release. -- internal: - - Fix possible data race in TestTracer. - - Pin versions of tools used for generation. - - Correct the release levels for BigQuery APIs. - - Start generation osconfig v1. -- longrunning: - - Add exported SetGoogleClientInfo wrappers to manual file. -- monitoring: - - Stop generation of monitoring/apiv3 because of incoming breaking change. -- trace: - - Add exported SetGoogleClientInfo wrappers to manual file. -- Various updates to autogenerated clients. - -## v0.56.0 - -- secretmanager: - - add IAM helper -- profiler: - - try all us-west1 zones for integration tests -- internal: - - add config to generate webrisk v1 - - add repo and commit to buildcop invocation - - add recaptchaenterprise v1 generation config - - update microgenerator to v0.12.5 - - add datacatalog client - - start generating security center settings v1beta - - start generating osconfig agentendpoint v1 - - setup generation for bigquery/connection/v1beta1 -- all: - - increase continous testing timeout to 45m - - various updates to autogenerated clients. - -## v0.55.0 - -- Various updates to autogenerated clients. - -## v0.54.0 - -- all: - - remove unused golang.org/x/exp from mod file - - update godoc.org links to pkg.go.dev -- compute/metadata: - - use defaultClient when http.Client is nil - - remove subscribeClient -- iam: - - add support for v3 policy and IAM conditions -- Various updates to autogenerated clients. - -## v0.53.0 - -- all: most clients now use transport/grpc.DialPool rather than Dial (see #1777 for outliers). - - Connection pooling now does not use the deprecated (and soon to be removed) gRPC load balancer API. -- profiler: remove symbolization (drops support for go1.10) -- Various updates to autogenerated clients. - -## v0.52.0 - -- internal/gapicgen: multiple improvements related to library generation. -- compute/metadata: unset ResponseHeaderTimeout in defaultClient -- docs: fix link to KMS in README.md -- Various updates to autogenerated clients. - -## v0.51.0 - -- secretmanager: - - add IAM helper for generic resource IAM handle -- cloudbuild: - - migrate to microgen in a major version -- Various updates to autogenerated clients. - -## v0.50.0 - -- profiler: - - Support disabling CPU profile collection. - - Log when a profile creation attempt begins. -- compute/metadata: - - Fix panic on malformed URLs. - - InstanceName returns actual instance name. -- Various updates to autogenerated clients. - -## v0.49.0 - -- functions/metadata: - - Handle string resources in JSON unmarshaller. -- Various updates to autogenerated clients. - -## v0.48.0 - -- Various updates to autogenerated clients - -## v0.47.0 - -This release drops support for Go 1.9 and Go 1.10: we continue to officially -support Go 1.11, Go 1.12, and Go 1.13. - -- Various updates to autogenerated clients. -- Add cloudbuild/apiv1 client. - -## v0.46.3 - -This is an empty release that was created solely to aid in storage's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.46.2 - -This is an empty release that was created solely to aid in spanner's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.46.1 - -This is an empty release that was created solely to aid in firestore's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.46.0 - -- spanner: - - Retry "Session not found" for read-only transactions. - - Retry aborted PDMLs. -- spanner/spannertest: - - Fix a bug that was causing 0X-prefixed number to be parsed incorrectly. -- storage: - - Add HMACKeyOptions. - - Remove *REGIONAL from StorageClass documentation. Using MULTI_REGIONAL, - DURABLE_REDUCED_AVAILABILITY, and REGIONAL are no longer best practice - StorageClasses but they are still acceptable values. -- trace: - - Remove cloud.google.com/go/trace. Package cloud.google.com/go/trace has been - marked OBSOLETE for several years: it is now no longer provided. If you - relied on this package, please vendor it or switch to using - https://cloud.google.com/trace/docs/setup/go (which obsoleted it). - -## v0.45.1 - -This is an empty release that was created solely to aid in pubsub's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.45.0 - -- compute/metadata: - - Add Email method. -- storage: - - Fix duplicated retry logic. - - Add ReaderObjectAttrs.StartOffset. - - Support reading last N bytes of a file when a negative range is given, such - as `obj.NewRangeReader(ctx, -10, -1)`. - - Add HMACKey listing functionality. -- spanner/spannertest: - - Support primary keys with no columns. - - Fix MinInt64 parsing. - - Implement deletion of key ranges. - - Handle reads during a read-write transaction. - - Handle returning DATE values. -- pubsub: - - Fix Ack/Modack request size calculation. -- logging: - - Add auto-detection of monitored resources on GAE Standard. - -## v0.44.3 - -This is an empty release that was created solely to aid in bigtable's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.44.2 - -This is an empty release that was created solely to aid in bigquery's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.44.1 - -This is an empty release that was created solely to aid in datastore's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.44.0 - -- datastore: - - Interface elements whose underlying types are supported, are now supported. - - Reduce time to initial retry from 1s to 100ms. -- firestore: - - Add Increment transformation. -- storage: - - Allow emulator with STORAGE_EMULATOR_HOST. - - Add methods for HMAC key management. -- pubsub: - - Add PublishCount and PublishLatency measurements. - - Add DefaultPublishViews and DefaultSubscribeViews for convenience of - importing all views. - - Add add Subscription.PushConfig.AuthenticationMethod. -- spanner: - - Allow emulator usage with SPANNER_EMULATOR_HOST. - - Add cloud.google.com/go/spanner/spannertest, a spanner emulator. - - Add cloud.google.com/go/spanner/spansql which contains types and a parser - for the Cloud Spanner SQL dialect. -- asset: - - Add apiv1p2beta1 client. - -## v0.43.0 - -This is an empty release that was created solely to aid in logging's module -carve-out. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository. - -## v0.42.0 - -- bigtable: - - Add an admin method to update an instance and clusters. - - Fix bttest regex matching behavior for alternations (things like `|a`). - - Expose BlockAllFilter filter. -- bigquery: - - Add Routines API support. -- storage: - - Add read-only Bucket.LocationType. -- logging: - - Add TraceSampled to Entry. - - Fix to properly extract {Trace, Span}Id from X-Cloud-Trace-Context. -- pubsub: - - Add Cloud Key Management to TopicConfig. - - Change ExpirationPolicy to optional.Duration. -- automl: - - Add apiv1beta1 client. -- iam: - - Fix compilation problem with iam/credentials/apiv1. - -## v0.41.0 - -- bigtable: - - Check results from PredicateFilter in bttest, which fixes certain false matches. -- profiler: - - debugLog checks user defined logging options before logging. -- spanner: - - PartitionedUpdates respect query parameters. - - StartInstance allows specifying cloud API access scopes. -- bigquery: - - Use empty slice instead of nil for ValueSaver, fixing an issue with zero-length, repeated, nested fields causing panics. -- firestore: - - Return same number of snapshots as doc refs (in the form of duplicate records) during GetAll. -- replay: - - Change references to IPv4 addresses to localhost, making replay compatible with IPv6. - -## v0.40.0 - -- all: - - Update to protobuf-golang v1.3.1. -- datastore: - - Attempt to decode GAE-encoded keys if initial decoding attempt fails. - - Support integer time conversion. -- pubsub: - - Add PublishSettings.BundlerByteLimit. If users receive pubsub.ErrOverflow, - this value should be adjusted higher. - - Use IPv6 compatible target in testutil. -- bigtable: - - Fix Latin-1 regexp filters in bttest, allowing \C. - - Expose PassAllFilter. -- profiler: - - Add log messages for slow path in start. - - Fix start to allow retry until success. -- firestore: - - Add admin client. -- containeranalysis: - - Add apiv1 client. -- grafeas: - - Add apiv1 client. - -## 0.39.0 - -- bigtable: - - Implement DeleteInstance in bttest. - - Return an error on invalid ReadRowsRequest.RowRange key ranges in bttest. -- bigquery: - - Move RequirePartitionFilter outside of TimePartioning. - - Expose models API. -- firestore: - - Allow array values in create and update calls. - - Add CollectionGroup method. -- pubsub: - - Add ExpirationPolicy to Subscription. -- storage: - - Add V4 signing. -- rpcreplay: - - Match streams by first sent request. This further improves rpcreplay's - ability to distinguish streams. -- httpreplay: - - Set up Man-In-The-Middle config only once. This should improve proxy - creation when multiple proxies are used in a single process. - - Remove error on empty Content-Type, allowing requests with no Content-Type - header but a non-empty body. -- all: - - Fix an edge case bug in auto-generated library pagination by properly - propagating pagetoken. - -## 0.38.0 - -This update includes a substantial reduction in our transitive dependency list -by way of updating to opencensus@v0.21.0. - -- spanner: - - Error implements GRPCStatus, allowing status.Convert. -- bigtable: - - Fix a bug in bttest that prevents single column queries returning results - that match other filters. - - Remove verbose retry logging. -- logging: - - Ensure RequestUrl has proper UTF-8, removing the need for users to wrap and - rune replace manually. -- recaptchaenterprise: - - Add v1beta1 client. -- phishingprotection: - - Add v1beta1 client. - -## 0.37.4 - -This patch releases re-builds the go.sum. This was not possible in the -previous release. - -- firestore: - - Add sentinel value DetectProjectID for auto-detecting project ID. - - Add OpenCensus tracing for public methods. - - Marked stable. All future changes come with a backwards compatibility - guarantee. - - Removed firestore/apiv1beta1. All users relying on this low-level library - should migrate to firestore/apiv1. Note that most users should use the - high-level firestore package instead. -- pubsub: - - Allow large messages in synchronous pull case. - - Cap bundler byte limit. This should prevent OOM conditions when there are - a very large number of message publishes occurring. -- storage: - - Add ETag to BucketAttrs and ObjectAttrs. -- datastore: - - Removed some non-sensical OpenCensus traces. -- webrisk: - - Add v1 client. -- asset: - - Add v1 client. -- cloudtasks: - - Add v2 client. - -## 0.37.3 - -This patch release removes github.com/golang/lint from the transitive -dependency list, resolving `go get -u` problems. - -Note: this release intentionally has a broken go.sum. Please use v0.37.4. - -## 0.37.2 - -This patch release is mostly intended to bring in v0.3.0 of -google.golang.org/api, which fixes a GCF deployment issue. - -Note: we had to-date accidentally marked Redis as stable. In this release, we've -fixed it by downgrading its documentation to alpha, as it is in other languages -and docs. - -- all: - - Document context in generated libraries. - -## 0.37.1 - -Small go.mod version bumps to bring in v0.2.0 of google.golang.org/api, which -introduces a new oauth2 url. - -## 0.37.0 - -- spanner: - - Add BatchDML method. - - Reduced initial time between retries. -- bigquery: - - Produce better error messages for InferSchema. - - Add logical type control for avro loads. - - Add support for the GEOGRAPHY type. -- datastore: - - Add sentinel value DetectProjectID for auto-detecting project ID. - - Allow flatten tag on struct pointers. - - Fixed a bug that caused queries to panic with invalid queries. Instead they - will now return an error. -- profiler: - - Add ability to override GCE zone and instance. -- pubsub: - - BEHAVIOR CHANGE: Refactor error code retry logic. RPCs should now more - consistently retry specific error codes based on whether they're idempotent - or non-idempotent. -- httpreplay: Fixed a bug when a non-GET request had a zero-length body causing - the Content-Length header to be dropped. -- iot: - - Add new apiv1 client. -- securitycenter: - - Add new apiv1 client. -- cloudscheduler: - - Add new apiv1 client. - -## 0.36.0 - -- spanner: - - Reduce minimum retry backoff from 1s to 100ms. This makes time between - retries much faster and should improve latency. -- storage: - - Add support for Bucket Policy Only. -- kms: - - Add ResourceIAM helper method. - - Deprecate KeyRingIAM and CryptoKeyIAM. Please use ResourceIAM. -- firestore: - - Switch from v1beta1 API to v1 API. - - Allow emulator with FIRESTORE_EMULATOR_HOST. -- bigquery: - - Add NumLongTermBytes to Table. - - Add TotalBytesProcessedAccuracy to QueryStatistics. -- irm: - - Add new v1alpha2 client. -- talent: - - Add new v4beta1 client. -- rpcreplay: - - Fix connection to work with grpc >= 1.17. - - It is now required for an actual gRPC server to be running for Dial to - succeed. - -## 0.35.1 - -- spanner: - - Adds OpenCensus views back to public API. - -## v0.35.0 - -- all: - - Add go.mod and go.sum. - - Switch usage of gax-go to gax-go/v2. -- bigquery: - - Fix bug where time partitioning could not be removed from a table. - - Fix panic that occurred with empty query parameters. -- bttest: - - Fix bug where deleted rows were returned by ReadRows. -- bigtable/emulator: - - Configure max message size to 256 MiB. -- firestore: - - Allow non-transactional queries in transactions. - - Allow StartAt/EndBefore on direct children at any depth. - - QuerySnapshotIterator.Stop may be called in an error state. - - Fix bug the prevented reset of transaction write state in between retries. -- functions/metadata: - - Make Metadata.Resource a pointer. -- logging: - - Make SpanID available in logging.Entry. -- metadata: - - Wrap !200 error code in a typed err. -- profiler: - - Add function to check if function name is within a particular file in the - profile. - - Set parent field in create profile request. - - Return kubernetes client to start cluster, so client can be used to poll - cluster. - - Add function for checking if filename is in profile. -- pubsub: - - Fix bug where messages expired without an initial modack in - synchronous=true mode. - - Receive does not retry ResourceExhausted errors. -- spanner: - - client.Close now cancels existing requests and should be much faster for - large amounts of sessions. - - Correctly allow MinOpened sessions to be spun up. - -## v0.34.0 - -- functions/metadata: - - Switch to using JSON in context. - - Make Resource a value. -- vision: Fix ProductSearch return type. -- datastore: Add an example for how to handle MultiError. - -## v0.33.1 - -- compute: Removes an erroneously added go.mod. -- logging: Populate source location in fromLogEntry. - -## v0.33.0 - -- bttest: - - Add support for apply_label_transformer. -- expr: - - Add expr library. -- firestore: - - Support retrieval of missing documents. -- kms: - - Add IAM methods. -- pubsub: - - Clarify extension documentation. -- scheduler: - - Add v1beta1 client. -- vision: - - Add product search helper. - - Add new product search client. - -## v0.32.0 - -Note: This release is the last to support Go 1.6 and 1.8. - -- bigquery: - - Add support for removing an expiration. - - Ignore NeverExpire in Table.Create. - - Validate table expiration time. -- cbt: - - Add note about not supporting arbitrary bytes. -- datastore: - - Align key checks. -- firestore: - - Return an error when using Start/End without providing values. -- pubsub: - - Add pstest Close method. - - Clarify MaxExtension documentation. -- securitycenter: - - Add v1beta1 client. -- spanner: - - Allow nil in mutations. - - Improve doc of SessionPoolConfig.MaxOpened. - - Increase session deletion timeout from 5s to 15s. - -## v0.31.0 - -- bigtable: - - Group mutations across multiple requests. -- bigquery: - - Link to bigquery troubleshooting errors page in bigquery.Error comment. -- cbt: - - Fix go generate command. - - Document usage of both maxage + maxversions. -- datastore: - - Passing nil keys results in ErrInvalidKey. -- firestore: - - Clarify what Document.DataTo does with untouched struct fields. -- profile: - - Validate service name in agent. -- pubsub: - - Fix deadlock with pstest and ctx.Cancel. - - Fix a possible deadlock in pstest. -- trace: - - Update doc URL with new fragment. - -Special thanks to @fastest963 for going above and beyond helping us to debug -hard-to-reproduce Pub/Sub issues. - -## v0.30.0 - -- spanner: DML support added. See https://godoc.org/cloud.google.com/go/spanner#hdr-DML_and_Partitioned_DML for more information. -- bigtable: bttest supports row sample filter. -- functions: metadata package added for accessing Cloud Functions resource metadata. - -## v0.29.0 - -- bigtable: - - Add retry to all idempotent RPCs. - - cbt supports complex GC policies. - - Emulator supports arbitrary bytes in regex filters. -- firestore: Add ArrayUnion and ArrayRemove. -- logging: Add the ContextFunc option to supply the context used for - asynchronous RPCs. -- profiler: Ignore NotDefinedError when fetching the instance name -- pubsub: - - BEHAVIOR CHANGE: Receive doesn't retry if an RPC returns codes.Cancelled. - - BEHAVIOR CHANGE: Receive retries on Unavailable intead of returning. - - Fix deadlock. - - Restore Ack/Nack/Modacks metrics. - - Improve context handling in iterator. - - Implement synchronous mode for Receive. - - pstest: add Pull. -- spanner: Add a metric for the number of sessions currently opened. -- storage: - - Canceling the context releases all resources. - - Add additional RetentionPolicy attributes. -- vision/apiv1: Add LocalizeObjects method. - -## v0.28.0 - -- bigtable: - - Emulator returns Unimplemented for snapshot RPCs. -- bigquery: - - Support zero-length repeated, nested fields. -- cloud assets: - - Add v1beta client. -- datastore: - - Don't nil out transaction ID on retry. -- firestore: - - BREAKING CHANGE: When watching a query with Query.Snapshots, QuerySnapshotIterator.Next - returns a QuerySnapshot which contains read time, result size, change list and the DocumentIterator - (previously, QuerySnapshotIterator.Next returned just the DocumentIterator). See: https://godoc.org/cloud.google.com/go/firestore#Query.Snapshots. - - Add array-contains operator. -- IAM: - - Add iam/credentials/apiv1 client. -- pubsub: - - Canceling the context passed to Subscription.Receive causes Receive to return when - processing finishes on all messages currently in progress, even if new messages are arriving. -- redis: - - Add redis/apiv1 client. -- storage: - - Add Reader.Attrs. - - Deprecate several Reader getter methods: please use Reader.Attrs for these instead. - - Add ObjectHandle.Bucket and ObjectHandle.Object methods. - -## v0.27.0 - -- bigquery: - - Allow modification of encryption configuration and partitioning options to a table via the Update call. - - Add a SchemaFromJSON function that converts a JSON table schema. -- bigtable: - - Restore cbt count functionality. -- containeranalysis: - - Add v1beta client. -- spanner: - - Fix a case where an iterator might not be closed correctly. -- storage: - - Add ServiceAccount method https://godoc.org/cloud.google.com/go/storage#Client.ServiceAccount. - - Add a method to Reader that returns the parsed value of the Last-Modified header. - -## v0.26.0 - -- bigquery: - - Support filtering listed jobs by min/max creation time. - - Support data clustering (https://godoc.org/cloud.google.com/go/bigquery#Clustering). - - Include job creator email in Job struct. -- bigtable: - - Add `RowSampleFilter`. - - emulator: BREAKING BEHAVIOR CHANGE: Regexps in row, family, column and value filters - must match the entire target string to succeed. Previously, the emulator was - succeeding on partial matches. - NOTE: As of this release, this change only affects the emulator when run - from this repo (bigtable/cmd/emulator/cbtemulator.go). The version launched - from `gcloud` will be updated in a subsequent `gcloud` release. -- dataproc: Add apiv1beta2 client. -- datastore: Save non-nil pointer fields on omitempty. -- logging: populate Entry.Trace from the HTTP X-Cloud-Trace-Context header. -- logging/logadmin: Support writer_identity and include_children. -- pubsub: - - Support labels on topics and subscriptions. - - Support message storage policy for topics. - - Use the distribution of ack times to determine when to extend ack deadlines. - The only user-visible effect of this change should be that programs that - call only `Subscription.Receive` need no IAM permissions other than `Pub/Sub - Subscriber`. -- storage: - - Support predefined ACLs. - - Support additional ACL fields other than Entity and Role. - - Support bucket websites. - - Support bucket logging. - - -## v0.25.0 - -- Added [Code of Conduct](https://github.com/googleapis/google-cloud-go/blob/master/CODE_OF_CONDUCT.md) -- bigtable: - - cbt: Support a GC policy of "never". -- errorreporting: - - Support User. - - Close now calls Flush. - - Use OnError (previously ignored). - - Pass through the RPC error as-is to OnError. -- httpreplay: A tool for recording and replaying HTTP requests - (for the bigquery and storage clients in this repo). -- kms: v1 client added -- logging: add SourceLocation to Entry. -- storage: improve CRC checking on read. - -## v0.24.0 - -- bigquery: Support for the NUMERIC type. -- bigtable: - - cbt: Optionally specify columns for read/lookup - - Support instance-level administration. -- oslogin: New client for the OS Login API. -- pubsub: - - The package is now stable. There will be no further breaking changes. - - Internal changes to improve Subscription.Receive behavior. -- storage: Support updating bucket lifecycle config. -- spanner: Support struct-typed parameter bindings. -- texttospeech: New client for the Text-to-Speech API. - -## v0.23.0 - -- bigquery: Add DDL stats to query statistics. -- bigtable: - - cbt: Add cells-per-column limit for row lookup. - - cbt: Make it possible to combine read filters. -- dlp: v2beta2 client removed. Use the v2 client instead. -- firestore, spanner: Fix compilation errors due to protobuf changes. - -## v0.22.0 - -- bigtable: - - cbt: Support cells per column limit for row read. - - bttest: Correctly handle empty RowSet. - - Fix ReadModifyWrite operation in emulator. - - Fix API path in GetCluster. - -- bigquery: - - BEHAVIOR CHANGE: Retry on 503 status code. - - Add dataset.DeleteWithContents. - - Add SchemaUpdateOptions for query jobs. - - Add Timeline to QueryStatistics. - - Add more stats to ExplainQueryStage. - - Support Parquet data format. - -- datastore: - - Support omitempty for times. - -- dlp: - - **BREAKING CHANGE:** Remove v1beta1 client. Please migrate to the v2 client, - which is now out of beta. - - Add v2 client. - -- firestore: - - BEHAVIOR CHANGE: Treat set({}, MergeAll) as valid. - -- iam: - - Support JWT signing via SignJwt callopt. - -- profiler: - - BEHAVIOR CHANGE: PollForSerialOutput returns an error when context.Done. - - BEHAVIOR CHANGE: Increase the initial backoff to 1 minute. - - Avoid returning empty serial port output. - -- pubsub: - - BEHAVIOR CHANGE: Don't backoff during next retryable error once stream is healthy. - - BEHAVIOR CHANGE: Don't backoff on EOF. - - pstest: Support Acknowledge and ModifyAckDeadline RPCs. - -- redis: - - Add v1 beta Redis client. - -- spanner: - - Support SessionLabels. - -- speech: - - Add api v1 beta1 client. - -- storage: - - BEHAVIOR CHANGE: Retry reads when retryable error occurs. - - Fix delete of object in requester-pays bucket. - - Support KMS integration. - -## v0.21.0 - -- bigquery: - - Add OpenCensus tracing. - -- firestore: - - **BREAKING CHANGE:** If a document does not exist, return a DocumentSnapshot - whose Exists method returns false. DocumentRef.Get and Transaction.Get - return the non-nil DocumentSnapshot in addition to a NotFound error. - **DocumentRef.GetAll and Transaction.GetAll return a non-nil - DocumentSnapshot instead of nil.** - - Add DocumentIterator.Stop. **Call Stop whenever you are done with a - DocumentIterator.** - - Added Query.Snapshots and DocumentRef.Snapshots, which provide realtime - notification of updates. See https://cloud.google.com/firestore/docs/query-data/listen. - - Canceling an RPC now always returns a grpc.Status with codes.Canceled. - -- spanner: - - Add `CommitTimestamp`, which supports inserting the commit timestamp of a - transaction into a column. - -## v0.20.0 - -- bigquery: Support SchemaUpdateOptions for load jobs. - -- bigtable: - - Add SampleRowKeys. - - cbt: Support union, intersection GCPolicy. - - Retry admin RPCS. - - Add trace spans to retries. - -- datastore: Add OpenCensus tracing. - -- firestore: - - Fix queries involving Null and NaN. - - Allow Timestamp protobuffers for time values. - -- logging: Add a WriteTimeout option. - -- spanner: Support Batch API. - -- storage: Add OpenCensus tracing. - -## v0.19.0 - -- bigquery: - - Support customer-managed encryption keys. - -- bigtable: - - Improved emulator support. - - Support GetCluster. - -- datastore: - - Add general mutations. - - Support pointer struct fields. - - Support transaction options. - -- firestore: - - Add Transaction.GetAll. - - Support document cursors. - -- logging: - - Support concurrent RPCs to the service. - - Support per-entry resources. - -- profiler: - - Add config options to disable heap and thread profiling. - - Read the project ID from $GOOGLE_CLOUD_PROJECT when it's set. - -- pubsub: - - BEHAVIOR CHANGE: Release flow control after ack/nack (instead of after the - callback returns). - - Add SubscriptionInProject. - - Add OpenCensus instrumentation for streaming pull. - -- storage: - - Support CORS. - -## v0.18.0 - -- bigquery: - - Marked stable. - - Schema inference of nullable fields supported. - - Added TimePartitioning to QueryConfig. - -- firestore: Data provided to DocumentRef.Set with a Merge option can contain - Delete sentinels. - -- logging: Clients can accept parent resources other than projects. - -- pubsub: - - pubsub/pstest: A lighweight fake for pubsub. Experimental; feedback welcome. - - Support updating more subscription metadata: AckDeadline, - RetainAckedMessages and RetentionDuration. - -- oslogin/apiv1beta: New client for the Cloud OS Login API. - -- rpcreplay: A package for recording and replaying gRPC traffic. - -- spanner: - - Add a ReadWithOptions that supports a row limit, as well as an index. - - Support query plan and execution statistics. - - Added [OpenCensus](http://opencensus.io) support. - -- storage: Clarify checksum validation for gzipped files (it is not validated - when the file is served uncompressed). - - -## v0.17.0 - -- firestore BREAKING CHANGES: - - Remove UpdateMap and UpdateStruct; rename UpdatePaths to Update. - Change - `docref.UpdateMap(ctx, map[string]interface{}{"a.b", 1})` - to - `docref.Update(ctx, []firestore.Update{{Path: "a.b", Value: 1}})` - - Change - `docref.UpdateStruct(ctx, []string{"Field"}, aStruct)` - to - `docref.Update(ctx, []firestore.Update{{Path: "Field", Value: aStruct.Field}})` - - Rename MergePaths to Merge; require args to be FieldPaths - - A value stored as an integer can be read into a floating-point field, and vice versa. -- bigtable/cmd/cbt: - - Support deleting a column. - - Add regex option for row read. -- spanner: Mark stable. -- storage: - - Add Reader.ContentEncoding method. - - Fix handling of SignedURL headers. -- bigquery: - - If Uploader.Put is called with no rows, it returns nil without making a - call. - - Schema inference supports the "nullable" option in struct tags for - non-required fields. - - TimePartitioning supports "Field". - - -## v0.16.0 - -- Other bigquery changes: - - `JobIterator.Next` returns `*Job`; removed `JobInfo` (BREAKING CHANGE). - - UseStandardSQL is deprecated; set UseLegacySQL to true if you need - Legacy SQL. - - Uploader.Put will generate a random insert ID if you do not provide one. - - Support time partitioning for load jobs. - - Support dry-run queries. - - A `Job` remembers its last retrieved status. - - Support retrieving job configuration. - - Support labels for jobs and tables. - - Support dataset access lists. - - Improve support for external data sources, including data from Bigtable and - Google Sheets, and tables with external data. - - Support updating a table's view configuration. - - Fix uploading civil times with nanoseconds. - -- storage: - - Support PubSub notifications. - - Support Requester Pays buckets. - -- profiler: Support goroutine and mutex profile types. - -## v0.15.0 - -- firestore: beta release. See the - [announcement](https://firebase.googleblog.com/2017/10/introducing-cloud-firestore.html). - -- errorreporting: The existing package has been redesigned. - -- errors: This package has been removed. Use errorreporting. - - -## v0.14.0 - -- bigquery BREAKING CHANGES: - - Standard SQL is the default for queries and views. - - `Table.Create` takes `TableMetadata` as a second argument, instead of - options. - - `Dataset.Create` takes `DatasetMetadata` as a second argument. - - `DatasetMetadata` field `ID` renamed to `FullID` - - `TableMetadata` field `ID` renamed to `FullID` - -- Other bigquery changes: - - The client will append a random suffix to a provided job ID if you set - `AddJobIDSuffix` to true in a job config. - - Listing jobs is supported. - - Better retry logic. - -- vision, language, speech: clients are now stable - -- monitoring: client is now beta - -- profiler: - - Rename InstanceName to Instance, ZoneName to Zone - - Auto-detect service name and version on AppEngine. - -## v0.13.0 - -- bigquery: UseLegacySQL options for CreateTable and QueryConfig. Use these - options to continue using Legacy SQL after the client switches its default - to Standard SQL. - -- bigquery: Support for updating dataset labels. - -- bigquery: Set DatasetIterator.ProjectID to list datasets in a project other - than the client's. DatasetsInProject is no longer needed and is deprecated. - -- bigtable: Fail ListInstances when any zones fail. - -- spanner: support decoding of slices of basic types (e.g. []string, []int64, - etc.) - -- logging/logadmin: UpdateSink no longer creates a sink if it is missing - (actually a change to the underlying service, not the client) - -- profiler: Service and ServiceVersion replace Target in Config. - -## v0.12.0 - -- pubsub: Subscription.Receive now uses streaming pull. - -- pubsub: add Client.TopicInProject to access topics in a different project - than the client. - -- errors: renamed errorreporting. The errors package will be removed shortly. - -- datastore: improved retry behavior. - -- bigquery: support updates to dataset metadata, with etags. - -- bigquery: add etag support to Table.Update (BREAKING: etag argument added). - -- bigquery: generate all job IDs on the client. - -- storage: support bucket lifecycle configurations. - - -## v0.11.0 - -- Clients for spanner, pubsub and video are now in beta. - -- New client for DLP. - -- spanner: performance and testing improvements. - -- storage: requester-pays buckets are supported. - -- storage, profiler, bigtable, bigquery: bug fixes and other minor improvements. - -- pubsub: bug fixes and other minor improvements - -## v0.10.0 - -- pubsub: Subscription.ModifyPushConfig replaced with Subscription.Update. - -- pubsub: Subscription.Receive now runs concurrently for higher throughput. - -- vision: cloud.google.com/go/vision is deprecated. Use -cloud.google.com/go/vision/apiv1 instead. - -- translation: now stable. - -- trace: several changes to the surface. See the link below. - -### Code changes required from v0.9.0 - -- pubsub: Replace - - ``` - sub.ModifyPushConfig(ctx, pubsub.PushConfig{Endpoint: "https://example.com/push"}) - ``` - - with - - ``` - sub.Update(ctx, pubsub.SubscriptionConfigToUpdate{ - PushConfig: &pubsub.PushConfig{Endpoint: "https://example.com/push"}, - }) - ``` - -- trace: traceGRPCServerInterceptor will be provided from *trace.Client. -Given an initialized `*trace.Client` named `tc`, instead of - - ``` - s := grpc.NewServer(grpc.UnaryInterceptor(trace.GRPCServerInterceptor(tc))) - ``` - - write - - ``` - s := grpc.NewServer(grpc.UnaryInterceptor(tc.GRPCServerInterceptor())) - ``` - -- trace trace.GRPCClientInterceptor will also provided from *trace.Client. -Instead of - - ``` - conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(trace.GRPCClientInterceptor())) - ``` - - write - - ``` - conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor())) - ``` - -- trace: We removed the deprecated `trace.EnableGRPCTracing`. Use the gRPC -interceptor as a dial option as shown below when initializing Cloud package -clients: - - ``` - c, err := pubsub.NewClient(ctx, "project-id", option.WithGRPCDialOption(grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor()))) - if err != nil { - ... - } - ``` - - -## v0.9.0 - -- Breaking changes to some autogenerated clients. -- rpcreplay package added. - -## v0.8.0 - -- profiler package added. -- storage: - - Retry Objects.Insert call. - - Add ProgressFunc to WRiter. -- pubsub: breaking changes: - - Publish is now asynchronous ([announcement](https://groups.google.com/d/topic/google-api-go-announce/aaqRDIQ3rvU/discussion)). - - Subscription.Pull replaced by Subscription.Receive, which takes a callback ([announcement](https://groups.google.com/d/topic/google-api-go-announce/8pt6oetAdKc/discussion)). - - Message.Done replaced with Message.Ack and Message.Nack. - -## v0.7.0 - -- Release of a client library for Spanner. See -the -[blog -post](https://cloudplatform.googleblog.com/2017/02/introducing-Cloud-Spanner-a-global-database-service-for-mission-critical-applications.html). -Note that although the Spanner service is beta, the Go client library is alpha. - -## v0.6.0 - -- Beta release of BigQuery, DataStore, Logging and Storage. See the -[blog post](https://cloudplatform.googleblog.com/2016/12/announcing-new-google-cloud-client.html). - -- bigquery: - - struct support. Read a row directly into a struct with -`RowIterator.Next`, and upload a row directly from a struct with `Uploader.Put`. -You can also use field tags. See the [package documentation][cloud-bigquery-ref] -for details. - - - The `ValueList` type was removed. It is no longer necessary. Instead of - ```go - var v ValueList - ... it.Next(&v) .. - ``` - use - - ```go - var v []Value - ... it.Next(&v) ... - ``` - - - Previously, repeatedly calling `RowIterator.Next` on the same `[]Value` or - `ValueList` would append to the slice. Now each call resets the size to zero first. - - - Schema inference will infer the SQL type BYTES for a struct field of - type []byte. Previously it inferred STRING. - - - The types `uint`, `uint64` and `uintptr` are no longer supported in schema - inference. BigQuery's integer type is INT64, and those types may hold values - that are not correctly represented in a 64-bit signed integer. - -## v0.5.0 - -- bigquery: - - The SQL types DATE, TIME and DATETIME are now supported. They correspond to - the `Date`, `Time` and `DateTime` types in the new `cloud.google.com/go/civil` - package. - - Support for query parameters. - - Support deleting a dataset. - - Values from INTEGER columns will now be returned as int64, not int. This - will avoid errors arising from large values on 32-bit systems. -- datastore: - - Nested Go structs encoded as Entity values, instead of a -flattened list of the embedded struct's fields. This means that you may now have twice-nested slices, eg. - ```go - type State struct { - Cities []struct{ - Populations []int - } - } - ``` - See [the announcement](https://groups.google.com/forum/#!topic/google-api-go-announce/79jtrdeuJAg) for -more details. - - Contexts no longer hold namespaces; instead you must set a key's namespace - explicitly. Also, key functions have been changed and renamed. - - The WithNamespace function has been removed. To specify a namespace in a Query, use the Query.Namespace method: - ```go - q := datastore.NewQuery("Kind").Namespace("ns") - ``` - - All the fields of Key are exported. That means you can construct any Key with a struct literal: - ```go - k := &Key{Kind: "Kind", ID: 37, Namespace: "ns"} - ``` - - As a result of the above, the Key methods Kind, ID, d.Name, Parent, SetParent and Namespace have been removed. - - `NewIncompleteKey` has been removed, replaced by `IncompleteKey`. Replace - ```go - NewIncompleteKey(ctx, kind, parent) - ``` - with - ```go - IncompleteKey(kind, parent) - ``` - and if you do use namespaces, make sure you set the namespace on the returned key. - - `NewKey` has been removed, replaced by `NameKey` and `IDKey`. Replace - ```go - NewKey(ctx, kind, name, 0, parent) - NewKey(ctx, kind, "", id, parent) - ``` - with - ```go - NameKey(kind, name, parent) - IDKey(kind, id, parent) - ``` - and if you do use namespaces, make sure you set the namespace on the returned key. - - The `Done` variable has been removed. Replace `datastore.Done` with `iterator.Done`, from the package `google.golang.org/api/iterator`. - - The `Client.Close` method will have a return type of error. It will return the result of closing the underlying gRPC connection. - - See [the announcement](https://groups.google.com/forum/#!topic/google-api-go-announce/hqXtM_4Ix-0) for -more details. - -## v0.4.0 - -- bigquery: - -`NewGCSReference` is now a function, not a method on `Client`. - - `Table.LoaderFrom` now accepts a `ReaderSource`, enabling - loading data into a table from a file or any `io.Reader`. - * Client.Table and Client.OpenTable have been removed. - Replace - ```go - client.OpenTable("project", "dataset", "table") - ``` - with - ```go - client.DatasetInProject("project", "dataset").Table("table") - ``` - - * Client.CreateTable has been removed. - Replace - ```go - client.CreateTable(ctx, "project", "dataset", "table") - ``` - with - ```go - client.DatasetInProject("project", "dataset").Table("table").Create(ctx) - ``` - - * Dataset.ListTables have been replaced with Dataset.Tables. - Replace - ```go - tables, err := ds.ListTables(ctx) - ``` - with - ```go - it := ds.Tables(ctx) - for { - table, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - // TODO: Handle error. - } - // TODO: use table. - } - ``` - - * Client.Read has been replaced with Job.Read, Table.Read and Query.Read. - Replace - ```go - it, err := client.Read(ctx, job) - ``` - with - ```go - it, err := job.Read(ctx) - ``` - and similarly for reading from tables or queries. - - * The iterator returned from the Read methods is now named RowIterator. Its - behavior is closer to the other iterators in these libraries. It no longer - supports the Schema method; see the next item. - Replace - ```go - for it.Next(ctx) { - var vals ValueList - if err := it.Get(&vals); err != nil { - // TODO: Handle error. - } - // TODO: use vals. - } - if err := it.Err(); err != nil { - // TODO: Handle error. - } - ``` - with - ``` - for { - var vals ValueList - err := it.Next(&vals) - if err == iterator.Done { - break - } - if err != nil { - // TODO: Handle error. - } - // TODO: use vals. - } - ``` - Instead of the `RecordsPerRequest(n)` option, write - ```go - it.PageInfo().MaxSize = n - ``` - Instead of the `StartIndex(i)` option, write - ```go - it.StartIndex = i - ``` - - * ValueLoader.Load now takes a Schema in addition to a slice of Values. - Replace - ```go - func (vl *myValueLoader) Load(v []bigquery.Value) - ``` - with - ```go - func (vl *myValueLoader) Load(v []bigquery.Value, s bigquery.Schema) - ``` - - - * Table.Patch is replace by Table.Update. - Replace - ```go - p := table.Patch() - p.Description("new description") - metadata, err := p.Apply(ctx) - ``` - with - ```go - metadata, err := table.Update(ctx, bigquery.TableMetadataToUpdate{ - Description: "new description", - }) - ``` - - * Client.Copy is replaced by separate methods for each of its four functions. - All options have been replaced by struct fields. - - * To load data from Google Cloud Storage into a table, use Table.LoaderFrom. - - Replace - ```go - client.Copy(ctx, table, gcsRef) - ``` - with - ```go - table.LoaderFrom(gcsRef).Run(ctx) - ``` - Instead of passing options to Copy, set fields on the Loader: - ```go - loader := table.LoaderFrom(gcsRef) - loader.WriteDisposition = bigquery.WriteTruncate - ``` - - * To extract data from a table into Google Cloud Storage, use - Table.ExtractorTo. Set fields on the returned Extractor instead of - passing options. - - Replace - ```go - client.Copy(ctx, gcsRef, table) - ``` - with - ```go - table.ExtractorTo(gcsRef).Run(ctx) - ``` - - * To copy data into a table from one or more other tables, use - Table.CopierFrom. Set fields on the returned Copier instead of passing options. - - Replace - ```go - client.Copy(ctx, dstTable, srcTable) - ``` - with - ```go - dst.Table.CopierFrom(srcTable).Run(ctx) - ``` - - * To start a query job, create a Query and call its Run method. Set fields - on the query instead of passing options. - - Replace - ```go - client.Copy(ctx, table, query) - ``` - with - ```go - query.Run(ctx) - ``` - - * Table.NewUploader has been renamed to Table.Uploader. Instead of options, - configure an Uploader by setting its fields. - Replace - ```go - u := table.NewUploader(bigquery.UploadIgnoreUnknownValues()) - ``` - with - ```go - u := table.NewUploader(bigquery.UploadIgnoreUnknownValues()) - u.IgnoreUnknownValues = true - ``` - -- pubsub: remove `pubsub.Done`. Use `iterator.Done` instead, where `iterator` is the package -`google.golang.org/api/iterator`. - -## v0.3.0 - -- storage: - * AdminClient replaced by methods on Client. - Replace - ```go - adminClient.CreateBucket(ctx, bucketName, attrs) - ``` - with - ```go - client.Bucket(bucketName).Create(ctx, projectID, attrs) - ``` - - * BucketHandle.List replaced by BucketHandle.Objects. - Replace - ```go - for query != nil { - objs, err := bucket.List(d.ctx, query) - if err != nil { ... } - query = objs.Next - for _, obj := range objs.Results { - fmt.Println(obj) - } - } - ``` - with - ```go - iter := bucket.Objects(d.ctx, query) - for { - obj, err := iter.Next() - if err == iterator.Done { - break - } - if err != nil { ... } - fmt.Println(obj) - } - ``` - (The `iterator` package is at `google.golang.org/api/iterator`.) - - Replace `Query.Cursor` with `ObjectIterator.PageInfo().Token`. - - Replace `Query.MaxResults` with `ObjectIterator.PageInfo().MaxSize`. - - - * ObjectHandle.CopyTo replaced by ObjectHandle.CopierFrom. - Replace - ```go - attrs, err := src.CopyTo(ctx, dst, nil) - ``` - with - ```go - attrs, err := dst.CopierFrom(src).Run(ctx) - ``` - - Replace - ```go - attrs, err := src.CopyTo(ctx, dst, &storage.ObjectAttrs{ContextType: "text/html"}) - ``` - with - ```go - c := dst.CopierFrom(src) - c.ContextType = "text/html" - attrs, err := c.Run(ctx) - ``` - - * ObjectHandle.ComposeFrom replaced by ObjectHandle.ComposerFrom. - Replace - ```go - attrs, err := dst.ComposeFrom(ctx, []*storage.ObjectHandle{src1, src2}, nil) - ``` - with - ```go - attrs, err := dst.ComposerFrom(src1, src2).Run(ctx) - ``` - - * ObjectHandle.Update's ObjectAttrs argument replaced by ObjectAttrsToUpdate. - Replace - ```go - attrs, err := obj.Update(ctx, &storage.ObjectAttrs{ContextType: "text/html"}) - ``` - with - ```go - attrs, err := obj.Update(ctx, storage.ObjectAttrsToUpdate{ContextType: "text/html"}) - ``` - - * ObjectHandle.WithConditions replaced by ObjectHandle.If. - Replace - ```go - obj.WithConditions(storage.Generation(gen), storage.IfMetaGenerationMatch(mgen)) - ``` - with - ```go - obj.Generation(gen).If(storage.Conditions{MetagenerationMatch: mgen}) - ``` - - Replace - ```go - obj.WithConditions(storage.IfGenerationMatch(0)) - ``` - with - ```go - obj.If(storage.Conditions{DoesNotExist: true}) - ``` - - * `storage.Done` replaced by `iterator.Done` (from package `google.golang.org/api/iterator`). - -- Package preview/logging deleted. Use logging instead. - -## v0.2.0 - -- Logging client replaced with preview version (see below). - -- New clients for some of Google's Machine Learning APIs: Vision, Speech, and -Natural Language. - -- Preview version of a new [Stackdriver Logging][cloud-logging] client in -[`cloud.google.com/go/preview/logging`](https://godoc.org/cloud.google.com/go/preview/logging). -This client uses gRPC as its transport layer, and supports log reading, sinks -and metrics. It will replace the current client at `cloud.google.com/go/logging` shortly. diff --git a/vendor/cloud.google.com/go/CODE_OF_CONDUCT.md b/vendor/cloud.google.com/go/CODE_OF_CONDUCT.md deleted file mode 100644 index 8fd1bc9c2..000000000 --- a/vendor/cloud.google.com/go/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,44 +0,0 @@ -# Contributor Code of Conduct - -As contributors and maintainers of this project, -and in the interest of fostering an open and welcoming community, -we pledge to respect all people who contribute through reporting issues, -posting feature requests, updating documentation, -submitting pull requests or patches, and other activities. - -We are committed to making participation in this project -a harassment-free experience for everyone, -regardless of level of experience, gender, gender identity and expression, -sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, religion, or nationality. - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery -* Personal attacks -* Trolling or insulting/derogatory comments -* Public or private harassment -* Publishing other's private information, -such as physical or electronic -addresses, without explicit permission -* Other unethical or unprofessional conduct. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct. -By adopting this Code of Conduct, -project maintainers commit themselves to fairly and consistently -applying these principles to every aspect of managing this project. -Project maintainers who do not follow or enforce the Code of Conduct -may be permanently removed from the project team. - -This code of conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. - -Instances of abusive, harassing, or otherwise unacceptable behavior -may be reported by opening an issue -or contacting one or more of the project maintainers. - -This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0, -available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/) - diff --git a/vendor/cloud.google.com/go/CONTRIBUTING.md b/vendor/cloud.google.com/go/CONTRIBUTING.md deleted file mode 100644 index 6d6e48b65..000000000 --- a/vendor/cloud.google.com/go/CONTRIBUTING.md +++ /dev/null @@ -1,327 +0,0 @@ -# Contributing - -1. [File an issue](https://github.com/googleapis/google-cloud-go/issues/new/choose). - The issue will be used to discuss the bug or feature and should be created - before sending a PR. - -1. [Install Go](https://golang.org/dl/). - 1. Ensure that your `GOBIN` directory (by default `$(go env GOPATH)/bin`) - is in your `PATH`. - 1. Check it's working by running `go version`. - * If it doesn't work, check the install location, usually - `/usr/local/go`, is on your `PATH`. - -1. Sign one of the -[contributor license agreements](#contributor-license-agreements) below. - -1. Clone the repo: - `git clone https://github.com/googleapis/google-cloud-go` - -1. Change into the checked out source: - `cd google-cloud-go` - -1. Fork the repo. - -1. Set your fork as a remote: - `git remote add fork git@github.com:GITHUB_USERNAME/google-cloud-go.git` - -1. Make changes, commit to your fork. - - Commit messages should follow the - [Conventional Commits Style](https://www.conventionalcommits.org). The scope - portion should always be filled with the name of the package affected by the - changes being made. For example: - ``` - feat(functions): add gophers codelab - ``` - -1. Send a pull request with your changes. - - To minimize friction, consider setting `Allow edits from maintainers` on the - PR, which will enable project committers and automation to update your PR. - -1. A maintainer will review the pull request and make comments. - - Prefer adding additional commits over amending and force-pushing since it can - be difficult to follow code reviews when the commit history changes. - - Commits will be squashed when they're merged. - -## Testing - -We test code against two versions of Go, the minimum and maximum versions -supported by our clients. To see which versions these are checkout our -[README](README.md#supported-versions). - -### Integration Tests - -In addition to the unit tests, you may run the integration test suite. These -directions describe setting up your environment to run integration tests for -_all_ packages: note that many of these instructions may be redundant if you -intend only to run integration tests on a single package. - -#### GCP Setup - -To run the integrations tests, creation and configuration of two projects in -the Google Developers Console is required: one specifically for Firestore -integration tests, and another for all other integration tests. We'll refer to -these projects as "general project" and "Firestore project". - -After creating each project, you must [create a service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount) -for each project. Ensure the project-level **Owner** -[IAM role](https://console.cloud.google.com/iam-admin/iam/project) role is added to -each service account. During the creation of the service account, you should -download the JSON credential file for use later. - -Next, ensure the following APIs are enabled in the general project: - -- BigQuery API -- BigQuery Data Transfer API -- Cloud Dataproc API -- Cloud Dataproc Control API Private -- Cloud Datastore API -- Cloud Firestore API -- Cloud Key Management Service (KMS) API -- Cloud Natural Language API -- Cloud OS Login API -- Cloud Pub/Sub API -- Cloud Resource Manager API -- Cloud Spanner API -- Cloud Speech API -- Cloud Translation API -- Cloud Video Intelligence API -- Cloud Vision API -- Compute Engine API -- Compute Engine Instance Group Manager API -- Container Registry API -- Firebase Rules API -- Google Cloud APIs -- Google Cloud Deployment Manager V2 API -- Google Cloud SQL -- Google Cloud Storage -- Google Cloud Storage JSON API -- Google Compute Engine Instance Group Updater API -- Google Compute Engine Instance Groups API -- Kubernetes Engine API -- Cloud Error Reporting API -- Pub/Sub Lite API - -Next, create a Datastore database in the general project, and a Firestore -database in the Firestore project. - -Finally, in the general project, create an API key for the translate API: - -- Go to GCP Developer Console. -- Navigate to APIs & Services > Credentials. -- Click Create Credentials > API Key. -- Save this key for use in `GCLOUD_TESTS_API_KEY` as described below. - -#### Local Setup - -Once the two projects are created and configured, set the following environment -variables: - -- `GCLOUD_TESTS_GOLANG_PROJECT_ID`: Developers Console project's ID (e.g. -bamboo-shift-455) for the general project. -- `GCLOUD_TESTS_GOLANG_KEY`: The path to the JSON key file of the general -project's service account. -- `GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID`: Developers Console project's ID -(e.g. doorway-cliff-677) for the Firestore project. -- `GCLOUD_TESTS_GOLANG_FIRESTORE_KEY`: The path to the JSON key file of the -Firestore project's service account. -- `GCLOUD_TESTS_API_KEY`: API key for using the Translate API created above. - -As part of the setup that follows, the following variables will be configured: - -- `GCLOUD_TESTS_GOLANG_KEYRING`: The full name of the keyring for the tests, -in the form -"projects/P/locations/L/keyRings/R". The creation of this is described below. -- `GCLOUD_TESTS_BIGTABLE_KEYRING`: The full name of the keyring for the bigtable tests, -in the form -"projects/P/locations/L/keyRings/R". The creation of this is described below. Expected to be single region. -- `GCLOUD_TESTS_GOLANG_ZONE`: Compute Engine zone. - -Install the [gcloud command-line tool][gcloudcli] to your machine and use it to -create some resources used in integration tests. - -From the project's root directory: - -``` sh -# Sets the default project in your env. -$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID - -# Authenticates the gcloud tool with your account. -$ gcloud auth login - -# Create the indexes used in the datastore integration tests. -$ gcloud datastore indexes create datastore/testdata/index.yaml - -# Creates a Google Cloud storage bucket with the same name as your test project, -# and with the Cloud Logging service account as owner, for the sink -# integration tests in logging. -$ gsutil mb gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID -$ gsutil acl ch -g cloud-logs@google.com:O gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID - -# Creates a PubSub topic for integration tests of storage notifications. -$ gcloud beta pubsub topics create go-storage-notification-test -# Next, go to the Pub/Sub dashboard in GCP console. Authorize the user -# "service-@gs-project-accounts.iam.gserviceaccount.com" -# as a publisher to that topic. - -# Creates a Spanner instance for the spanner integration tests. -$ gcloud beta spanner instances create go-integration-test --config regional-us-central1 --nodes 10 --description 'Instance for go client test' -# NOTE: Spanner instances are priced by the node-hour, so you may want to -# delete the instance after testing with 'gcloud beta spanner instances delete'. - -$ export MY_KEYRING=some-keyring-name -$ export MY_LOCATION=global -$ export MY_SINGLE_LOCATION=us-central1 -# Creates a KMS keyring, in the same location as the default location for your -# project's buckets. -$ gcloud kms keyrings create $MY_KEYRING --location $MY_LOCATION -# Creates two keys in the keyring, named key1 and key2. -$ gcloud kms keys create key1 --keyring $MY_KEYRING --location $MY_LOCATION --purpose encryption -$ gcloud kms keys create key2 --keyring $MY_KEYRING --location $MY_LOCATION --purpose encryption -# Sets the GCLOUD_TESTS_GOLANG_KEYRING environment variable. -$ export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_LOCATION/keyRings/$MY_KEYRING -# Authorizes Google Cloud Storage to encrypt and decrypt using key1. -$ gsutil kms authorize -p $GCLOUD_TESTS_GOLANG_PROJECT_ID -k $GCLOUD_TESTS_GOLANG_KEYRING/cryptoKeys/key1 - -# Create KMS Key in one region for Bigtable -$ gcloud kms keyrings create $MY_KEYRING --location $MY_SINGLE_LOCATION -$ gcloud kms keys create key1 --keyring $MY_KEYRING --location $MY_SINGLE_LOCATION --purpose encryption -# Sets the GCLOUD_TESTS_BIGTABLE_KEYRING environment variable. -$ export GCLOUD_TESTS_BIGTABLE_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_SINGLE_LOCATION/keyRings/$MY_KEYRING -# Create a service agent, https://cloud.google.com/bigtable/docs/use-cmek#gcloud: -$ gcloud beta services identity create \ - --service=bigtableadmin.googleapis.com \ - --project $GCLOUD_TESTS_GOLANG_PROJECT_ID -# Note the service agent email for the agent created. -$ export SERVICE_AGENT_EMAIL= - -# Authorizes Google Cloud Bigtable to encrypt and decrypt using key1 -$ gcloud kms keys add-iam-policy-binding key1 \ - --keyring $MY_KEYRING \ - --location $MY_SINGLE_LOCATION \ - --role roles/cloudkms.cryptoKeyEncrypterDecrypter \ - --member "serviceAccount:$SERVICE_AGENT_EMAIL" \ - --project $GCLOUD_TESTS_GOLANG_PROJECT_ID -``` - -It may be useful to add exports to your shell initialization for future use. -For instance, in `.zshrc`: - -```sh -#### START GO SDK Test Variables -# Developers Console project's ID (e.g. bamboo-shift-455) for the general project. -export GCLOUD_TESTS_GOLANG_PROJECT_ID=your-project - -# The path to the JSON key file of the general project's service account. -export GCLOUD_TESTS_GOLANG_KEY=~/directory/your-project-abcd1234.json - -# Developers Console project's ID (e.g. doorway-cliff-677) for the Firestore project. -export GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID=your-firestore-project - -# The path to the JSON key file of the Firestore project's service account. -export GCLOUD_TESTS_GOLANG_FIRESTORE_KEY=~/directory/your-firestore-project-abcd1234.json - -# The full name of the keyring for the tests, in the form "projects/P/locations/L/keyRings/R". -# The creation of this is described below. -export MY_KEYRING=my-golang-sdk-test -export MY_LOCATION=global -export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_LOCATION/keyRings/$MY_KEYRING - -# API key for using the Translate API. -export GCLOUD_TESTS_API_KEY=abcdefghijk123456789 - -# Compute Engine zone. (https://cloud.google.com/compute/docs/regions-zones) -export GCLOUD_TESTS_GOLANG_ZONE=your-chosen-region -#### END GO SDK Test Variables -``` - -#### Running - -Once you've done the necessary setup, you can run the integration tests by -running: - -``` sh -$ go test -v ./... -``` - -Note that the above command will not run the tests in other modules. To run -tests on other modules, first navigate to the appropriate -subdirectory. For instance, to run only the tests for datastore: -``` sh -$ cd datastore -$ go test -v ./... -``` - -#### Replay - -Some packages can record the RPCs during integration tests to a file for -subsequent replay. To record, pass the `-record` flag to `go test`. The -recording will be saved to the _package_`.replay` file. To replay integration -tests from a saved recording, the replay file must be present, the `-short` -flag must be passed to `go test`, and the `GCLOUD_TESTS_GOLANG_ENABLE_REPLAY` -environment variable must have a non-empty value. - -## Contributor License Agreements - -Before we can accept your pull requests you'll need to sign a Contributor -License Agreement (CLA): - -- **If you are an individual writing original source code** and **you own the -intellectual property**, then you'll need to sign an [individual CLA][indvcla]. -- **If you work for a company that wants to allow you to contribute your -work**, then you'll need to sign a [corporate CLA][corpcla]. - -You can sign these electronically (just scroll to the bottom). After that, -we'll be able to accept your pull requests. - -## Contributor Code of Conduct - -As contributors and maintainers of this project, -and in the interest of fostering an open and welcoming community, -we pledge to respect all people who contribute through reporting issues, -posting feature requests, updating documentation, -submitting pull requests or patches, and other activities. - -We are committed to making participation in this project -a harassment-free experience for everyone, -regardless of level of experience, gender, gender identity and expression, -sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, religion, or nationality. - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery -* Personal attacks -* Trolling or insulting/derogatory comments -* Public or private harassment -* Publishing other's private information, -such as physical or electronic -addresses, without explicit permission -* Other unethical or unprofessional conduct. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct. -By adopting this Code of Conduct, -project maintainers commit themselves to fairly and consistently -applying these principles to every aspect of managing this project. -Project maintainers who do not follow or enforce the Code of Conduct -may be permanently removed from the project team. - -This code of conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. - -Instances of abusive, harassing, or otherwise unacceptable behavior -may be reported by opening an issue -or contacting one or more of the project maintainers. - -This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.2.0, -available at [https://contributor-covenant.org/version/1/2/0/](https://contributor-covenant.org/version/1/2/0/) - -[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/ -[indvcla]: https://developers.google.com/open-source/cla/individual -[corpcla]: https://developers.google.com/open-source/cla/corporate diff --git a/vendor/cloud.google.com/go/LICENSE b/vendor/cloud.google.com/go/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/vendor/cloud.google.com/go/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/cloud.google.com/go/README.md b/vendor/cloud.google.com/go/README.md deleted file mode 100644 index 01453cc69..000000000 --- a/vendor/cloud.google.com/go/README.md +++ /dev/null @@ -1,139 +0,0 @@ -# Google Cloud Client Libraries for Go - -[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go.svg)](https://pkg.go.dev/cloud.google.com/go) - -Go packages for [Google Cloud Platform](https://cloud.google.com) services. - -``` go -import "cloud.google.com/go" -``` - -To install the packages on your system, *do not clone the repo*. Instead: - -1. Change to your project directory: - - ```bash - cd /my/cloud/project - ``` -1. Get the package you want to use. Some products have their own module, so it's - best to `go get` the package(s) you want to use: - - ``` - $ go get cloud.google.com/go/firestore # Replace with the package you want to use. - ``` - -**NOTE:** Some of these packages are under development, and may occasionally -make backwards-incompatible changes. - -## Supported APIs - -For an updated list of all of our released APIs please see our -[reference docs](https://cloud.google.com/go/docs/reference). - -## [Go Versions Supported](#supported-versions) - -Our libraries are compatible with at least the three most recent, major Go -releases. They are currently compatible with: - -- Go 1.19 -- Go 1.18 -- Go 1.17 -- Go 1.16 -- Go 1.15 - -## Authorization - -By default, each API will use [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials) -for authorization credentials used in calling the API endpoints. This will allow your -application to run in many environments without requiring explicit configuration. - -[snip]:# (auth) -```go -client, err := storage.NewClient(ctx) -``` - -To authorize using a -[JSON key file](https://cloud.google.com/iam/docs/managing-service-account-keys), -pass -[`option.WithCredentialsFile`](https://pkg.go.dev/google.golang.org/api/option#WithCredentialsFile) -to the `NewClient` function of the desired package. For example: - -[snip]:# (auth-JSON) -```go -client, err := storage.NewClient(ctx, option.WithCredentialsFile("path/to/keyfile.json")) -``` - -You can exert more control over authorization by using the -[`golang.org/x/oauth2`](https://pkg.go.dev/golang.org/x/oauth2) package to -create an `oauth2.TokenSource`. Then pass -[`option.WithTokenSource`](https://pkg.go.dev/google.golang.org/api/option#WithTokenSource) -to the `NewClient` function: -[snip]:# (auth-ts) -```go -tokenSource := ... -client, err := storage.NewClient(ctx, option.WithTokenSource(tokenSource)) -``` - -## Contributing - -Contributions are welcome. Please, see the -[CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) -document for details. - -Please note that this project is released with a Contributor Code of Conduct. -By participating in this project you agree to abide by its terms. -See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) -for more information. - -[cloud-asset]: https://cloud.google.com/security-command-center/docs/how-to-asset-inventory -[cloud-automl]: https://cloud.google.com/automl -[cloud-build]: https://cloud.google.com/cloud-build/ -[cloud-bigquery]: https://cloud.google.com/bigquery/ -[cloud-bigtable]: https://cloud.google.com/bigtable/ -[cloud-compute]: https://cloud.google.com/compute -[cloud-container]: https://cloud.google.com/containers/ -[cloud-containeranalysis]: https://cloud.google.com/container-registry/docs/container-analysis -[cloud-dataproc]: https://cloud.google.com/dataproc/ -[cloud-datastore]: https://cloud.google.com/datastore/ -[cloud-dialogflow]: https://cloud.google.com/dialogflow-enterprise/ -[cloud-debugger]: https://cloud.google.com/debugger/ -[cloud-dlp]: https://cloud.google.com/dlp/ -[cloud-errors]: https://cloud.google.com/error-reporting/ -[cloud-firestore]: https://cloud.google.com/firestore/ -[cloud-iam]: https://cloud.google.com/iam/ -[cloud-iot]: https://cloud.google.com/iot-core/ -[cloud-irm]: https://cloud.google.com/incident-response/docs/concepts -[cloud-kms]: https://cloud.google.com/kms/ -[cloud-pubsub]: https://cloud.google.com/pubsub/ -[cloud-pubsublite]: https://cloud.google.com/pubsub/lite -[cloud-storage]: https://cloud.google.com/storage/ -[cloud-language]: https://cloud.google.com/natural-language -[cloud-logging]: https://cloud.google.com/logging/ -[cloud-natural-language]: https://cloud.google.com/natural-language/ -[cloud-memorystore]: https://cloud.google.com/memorystore/ -[cloud-monitoring]: https://cloud.google.com/monitoring/ -[cloud-oslogin]: https://cloud.google.com/compute/docs/oslogin/rest -[cloud-phishingprotection]: https://cloud.google.com/phishing-protection/ -[cloud-securitycenter]: https://cloud.google.com/security-command-center/ -[cloud-scheduler]: https://cloud.google.com/scheduler -[cloud-spanner]: https://cloud.google.com/spanner/ -[cloud-speech]: https://cloud.google.com/speech -[cloud-talent]: https://cloud.google.com/solutions/talent-solution/ -[cloud-tasks]: https://cloud.google.com/tasks/ -[cloud-texttospeech]: https://cloud.google.com/texttospeech/ -[cloud-talent]: https://cloud.google.com/solutions/talent-solution/ -[cloud-trace]: https://cloud.google.com/trace/ -[cloud-translate]: https://cloud.google.com/translate -[cloud-recaptcha]: https://cloud.google.com/recaptcha-enterprise/ -[cloud-recommender]: https://cloud.google.com/recommendations/ -[cloud-video]: https://cloud.google.com/video-intelligence/ -[cloud-vision]: https://cloud.google.com/vision -[cloud-webrisk]: https://cloud.google.com/web-risk/ - -## Links - -- [Go on Google Cloud](https://cloud.google.com/go/home) -- [Getting started with Go on Google Cloud](https://cloud.google.com/go/getting-started) -- [App Engine Quickstart](https://cloud.google.com/appengine/docs/standard/go/quickstart) -- [Cloud Functions Quickstart](https://cloud.google.com/functions/docs/quickstart-go) -- [Cloud Run Quickstart](https://cloud.google.com/run/docs/quickstarts/build-and-deploy#go) diff --git a/vendor/cloud.google.com/go/RELEASING.md b/vendor/cloud.google.com/go/RELEASING.md deleted file mode 100644 index 6d0fcf4f9..000000000 --- a/vendor/cloud.google.com/go/RELEASING.md +++ /dev/null @@ -1,141 +0,0 @@ -# Releasing - -## Determine which module to release - -The Go client libraries have several modules. Each module does not strictly -correspond to a single library - they correspond to trees of directories. If a -file needs to be released, you must release the closest ancestor module. - -To see all modules: - -```bash -$ cat `find . -name go.mod` | grep module -module cloud.google.com/go/pubsub -module cloud.google.com/go/spanner -module cloud.google.com/go -module cloud.google.com/go/bigtable -module cloud.google.com/go/bigquery -module cloud.google.com/go/storage -module cloud.google.com/go/pubsublite -module cloud.google.com/go/firestore -module cloud.google.com/go/logging -module cloud.google.com/go/internal/gapicgen -module cloud.google.com/go/internal/godocfx -module cloud.google.com/go/internal/examples/fake -module cloud.google.com/go/internal/examples/mock -module cloud.google.com/go/datastore -``` - -The `cloud.google.com/go` is the repository root module. Each other module is -a submodule. - -So, if you need to release a change in `bigtable/bttest/inmem.go`, the closest -ancestor module is `cloud.google.com/go/bigtable` - so you should release a new -version of the `cloud.google.com/go/bigtable` submodule. - -If you need to release a change in `asset/apiv1/asset_client.go`, the closest -ancestor module is `cloud.google.com/go` - so you should release a new version -of the `cloud.google.com/go` repository root module. Note: releasing -`cloud.google.com/go` has no impact on any of the submodules, and vice-versa. -They are released entirely independently. - -## Test failures - -If there are any test failures in the Kokoro build, releases are blocked until -the failures have been resolved. - -## How to release - -### Automated Releases (`cloud.google.com/go` and submodules) - -We now use [release-please](https://github.com/googleapis/release-please) to -perform automated releases for `cloud.google.com/go` and all submodules. - -1. If there are changes that have not yet been released, a - [pull request](https://github.com/googleapis/google-cloud-go/pull/2971) will - be automatically opened by release-please - with a title like "chore: release X.Y.Z" (for the root module) or - "chore: release datastore X.Y.Z" (for the datastore submodule), where X.Y.Z - is the next version to be released. Find the desired pull request - [here](https://github.com/googleapis/google-cloud-go/pulls) -1. Check for failures in the - [continuous Kokoro build](http://go/google-cloud-go-continuous). If there are - any failures in the most recent build, address them before proceeding with - the release. (This applies even if the failures are in a different submodule - from the one being released.) -1. Review the release notes. These are automatically generated from the titles - of any merged commits since the previous release. If you would like to edit - them, this can be done by updating the changes in the release PR. -1. To cut a release, approve and merge the pull request. Doing so will - update the `CHANGES.md`, tag the merged commit with the appropriate version, - and draft a GitHub release which will copy the notes from `CHANGES.md`. - -### Manual Release (`cloud.google.com/go`) - -If for whatever reason the automated release process is not working as expected, -here is how to manually cut a release of `cloud.google.com/go`. - -1. Check for failures in the - [continuous Kokoro build](http://go/google-cloud-go-continuous). If there are - any failures in the most recent build, address them before proceeding with - the release. -1. Navigate to `google-cloud-go/` and switch to main. -1. `git pull` -1. Run `git tag -l | grep -v beta | grep -v alpha` to see all existing releases. - The current latest tag `$CV` is the largest tag. It should look something - like `vX.Y.Z` (note: ignore all `LIB/vX.Y.Z` tags - these are tags for a - specific library, not the module root). We'll call the current version `$CV` - and the new version `$NV`. -1. On main, run `git log $CV...` to list all the changes since the last - release. NOTE: You must manually visually parse out changes to submodules [1] - (the `git log` is going to show you things in submodules, which are not going - to be part of your release). -1. Edit `CHANGES.md` to include a summary of the changes. -1. In `internal/version/version.go`, update `const Repo` to today's date with - the format `YYYYMMDD`. -1. In `internal/version` run `go generate`. -1. Commit the changes, ignoring the generated `.go-r` file. Push to your fork, - and create a PR titled `chore: release $NV`. -1. Wait for the PR to be reviewed and merged. Once it's merged, and without - merging any other PRs in the meantime: - a. Switch to main. - b. `git pull` - c. Tag the repo with the next version: `git tag $NV`. - d. Push the tag to origin: - `git push origin $NV` -1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) - with the new release, copying the contents of `CHANGES.md`. - -### Manual Releases (submodules) - -If for whatever reason the automated release process is not working as expected, -here is how to manually cut a release of a submodule. - -(these instructions assume we're releasing `cloud.google.com/go/datastore` - adjust accordingly) - -1. Check for failures in the - [continuous Kokoro build](http://go/google-cloud-go-continuous). If there are - any failures in the most recent build, address them before proceeding with - the release. (This applies even if the failures are in a different submodule - from the one being released.) -1. Navigate to `google-cloud-go/` and switch to main. -1. `git pull` -1. Run `git tag -l | grep datastore | grep -v beta | grep -v alpha` to see all - existing releases. The current latest tag `$CV` is the largest tag. It - should look something like `datastore/vX.Y.Z`. We'll call the current version - `$CV` and the new version `$NV`. -1. On main, run `git log $CV.. -- datastore/` to list all the changes to the - submodule directory since the last release. -1. Edit `datastore/CHANGES.md` to include a summary of the changes. -1. In `internal/version` run `go generate`. -1. Commit the changes, ignoring the generated `.go-r` file. Push to your fork, - and create a PR titled `chore(datastore): release $NV`. -1. Wait for the PR to be reviewed and merged. Once it's merged, and without - merging any other PRs in the meantime: - a. Switch to main. - b. `git pull` - c. Tag the repo with the next version: `git tag $NV`. - d. Push the tag to origin: - `git push origin $NV` -1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) - with the new release, copying the contents of `datastore/CHANGES.md`. diff --git a/vendor/cloud.google.com/go/SECURITY.md b/vendor/cloud.google.com/go/SECURITY.md deleted file mode 100644 index 8b58ae9c0..000000000 --- a/vendor/cloud.google.com/go/SECURITY.md +++ /dev/null @@ -1,7 +0,0 @@ -# Security Policy - -To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). - -The Google Security Team will respond within 5 working days of your report on g.co/vulnz. - -We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/vendor/cloud.google.com/go/compute/LICENSE b/vendor/cloud.google.com/go/compute/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/vendor/cloud.google.com/go/compute/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/cloud.google.com/go/compute/internal/version.go b/vendor/cloud.google.com/go/compute/internal/version.go deleted file mode 100644 index efedadbea..000000000 --- a/vendor/cloud.google.com/go/compute/internal/version.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -// Version is the current tagged release of the library. -const Version = "1.13.0" diff --git a/vendor/cloud.google.com/go/compute/metadata/CHANGES.md b/vendor/cloud.google.com/go/compute/metadata/CHANGES.md deleted file mode 100644 index 8631b6d6d..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/CHANGES.md +++ /dev/null @@ -1,5 +0,0 @@ -# Changes - -## [0.1.0] (2022-10-26) - -Initial release of metadata being it's own module. diff --git a/vendor/cloud.google.com/go/compute/metadata/LICENSE b/vendor/cloud.google.com/go/compute/metadata/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/cloud.google.com/go/compute/metadata/README.md b/vendor/cloud.google.com/go/compute/metadata/README.md deleted file mode 100644 index f940fb2c8..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Compute API - -[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/compute.svg)](https://pkg.go.dev/cloud.google.com/go/compute/metadata) - -This is a utility library for communicating with Google Cloud metadata service -on Google Cloud. - -## Install - -```bash -go get cloud.google.com/go/compute/metadata -``` - -## Go Version Support - -See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) -section in the root directory's README. - -## Contributing - -Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) -document for details. - -Please note that this project is released with a Contributor Code of Conduct. -By participating in this project you agree to abide by its terms. See -[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) -for more information. diff --git a/vendor/cloud.google.com/go/compute/metadata/metadata.go b/vendor/cloud.google.com/go/compute/metadata/metadata.go deleted file mode 100644 index 50538b1d3..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/metadata.go +++ /dev/null @@ -1,542 +0,0 @@ -// Copyright 2014 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package metadata provides access to Google Compute Engine (GCE) -// metadata and API service accounts. -// -// This package is a wrapper around the GCE metadata service, -// as documented at https://cloud.google.com/compute/docs/metadata/overview. -package metadata // import "cloud.google.com/go/compute/metadata" - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "os" - "runtime" - "strings" - "sync" - "time" -) - -const ( - // metadataIP is the documented metadata server IP address. - metadataIP = "169.254.169.254" - - // metadataHostEnv is the environment variable specifying the - // GCE metadata hostname. If empty, the default value of - // metadataIP ("169.254.169.254") is used instead. - // This is variable name is not defined by any spec, as far as - // I know; it was made up for the Go package. - metadataHostEnv = "GCE_METADATA_HOST" - - userAgent = "gcloud-golang/0.1" -) - -type cachedValue struct { - k string - trim bool - mu sync.Mutex - v string -} - -var ( - projID = &cachedValue{k: "project/project-id", trim: true} - projNum = &cachedValue{k: "project/numeric-project-id", trim: true} - instID = &cachedValue{k: "instance/id", trim: true} -) - -var defaultClient = &Client{hc: newDefaultHTTPClient()} - -func newDefaultHTTPClient() *http.Client { - return &http.Client{ - Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 2 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - }, - Timeout: 5 * time.Second, - } -} - -// NotDefinedError is returned when requested metadata is not defined. -// -// The underlying string is the suffix after "/computeMetadata/v1/". -// -// This error is not returned if the value is defined to be the empty -// string. -type NotDefinedError string - -func (suffix NotDefinedError) Error() string { - return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix)) -} - -func (c *cachedValue) get(cl *Client) (v string, err error) { - defer c.mu.Unlock() - c.mu.Lock() - if c.v != "" { - return c.v, nil - } - if c.trim { - v, err = cl.getTrimmed(c.k) - } else { - v, err = cl.Get(c.k) - } - if err == nil { - c.v = v - } - return -} - -var ( - onGCEOnce sync.Once - onGCE bool -) - -// OnGCE reports whether this process is running on Google Compute Engine. -func OnGCE() bool { - onGCEOnce.Do(initOnGCE) - return onGCE -} - -func initOnGCE() { - onGCE = testOnGCE() -} - -func testOnGCE() bool { - // The user explicitly said they're on GCE, so trust them. - if os.Getenv(metadataHostEnv) != "" { - return true - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - resc := make(chan bool, 2) - - // Try two strategies in parallel. - // See https://github.com/googleapis/google-cloud-go/issues/194 - go func() { - req, _ := http.NewRequest("GET", "http://"+metadataIP, nil) - req.Header.Set("User-Agent", userAgent) - res, err := newDefaultHTTPClient().Do(req.WithContext(ctx)) - if err != nil { - resc <- false - return - } - defer res.Body.Close() - resc <- res.Header.Get("Metadata-Flavor") == "Google" - }() - - go func() { - resolver := &net.Resolver{} - addrs, err := resolver.LookupHost(ctx, "metadata.google.internal") - if err != nil || len(addrs) == 0 { - resc <- false - return - } - resc <- strsContains(addrs, metadataIP) - }() - - tryHarder := systemInfoSuggestsGCE() - if tryHarder { - res := <-resc - if res { - // The first strategy succeeded, so let's use it. - return true - } - // Wait for either the DNS or metadata server probe to - // contradict the other one and say we are running on - // GCE. Give it a lot of time to do so, since the system - // info already suggests we're running on a GCE BIOS. - timer := time.NewTimer(5 * time.Second) - defer timer.Stop() - select { - case res = <-resc: - return res - case <-timer.C: - // Too slow. Who knows what this system is. - return false - } - } - - // There's no hint from the system info that we're running on - // GCE, so use the first probe's result as truth, whether it's - // true or false. The goal here is to optimize for speed for - // users who are NOT running on GCE. We can't assume that - // either a DNS lookup or an HTTP request to a blackholed IP - // address is fast. Worst case this should return when the - // metaClient's Transport.ResponseHeaderTimeout or - // Transport.Dial.Timeout fires (in two seconds). - return <-resc -} - -// systemInfoSuggestsGCE reports whether the local system (without -// doing network requests) suggests that we're running on GCE. If this -// returns true, testOnGCE tries a bit harder to reach its metadata -// server. -func systemInfoSuggestsGCE() bool { - if runtime.GOOS != "linux" { - // We don't have any non-Linux clues available, at least yet. - return false - } - slurp, _ := ioutil.ReadFile("/sys/class/dmi/id/product_name") - name := strings.TrimSpace(string(slurp)) - return name == "Google" || name == "Google Compute Engine" -} - -// Subscribe calls Client.Subscribe on the default client. -func Subscribe(suffix string, fn func(v string, ok bool) error) error { - return defaultClient.Subscribe(suffix, fn) -} - -// Get calls Client.Get on the default client. -func Get(suffix string) (string, error) { return defaultClient.Get(suffix) } - -// ProjectID returns the current instance's project ID string. -func ProjectID() (string, error) { return defaultClient.ProjectID() } - -// NumericProjectID returns the current instance's numeric project ID. -func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() } - -// InternalIP returns the instance's primary internal IP address. -func InternalIP() (string, error) { return defaultClient.InternalIP() } - -// ExternalIP returns the instance's primary external (public) IP address. -func ExternalIP() (string, error) { return defaultClient.ExternalIP() } - -// Email calls Client.Email on the default client. -func Email(serviceAccount string) (string, error) { return defaultClient.Email(serviceAccount) } - -// Hostname returns the instance's hostname. This will be of the form -// ".c..internal". -func Hostname() (string, error) { return defaultClient.Hostname() } - -// InstanceTags returns the list of user-defined instance tags, -// assigned when initially creating a GCE instance. -func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() } - -// InstanceID returns the current VM's numeric instance ID. -func InstanceID() (string, error) { return defaultClient.InstanceID() } - -// InstanceName returns the current VM's instance ID string. -func InstanceName() (string, error) { return defaultClient.InstanceName() } - -// Zone returns the current VM's zone, such as "us-central1-b". -func Zone() (string, error) { return defaultClient.Zone() } - -// InstanceAttributes calls Client.InstanceAttributes on the default client. -func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() } - -// ProjectAttributes calls Client.ProjectAttributes on the default client. -func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() } - -// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client. -func InstanceAttributeValue(attr string) (string, error) { - return defaultClient.InstanceAttributeValue(attr) -} - -// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client. -func ProjectAttributeValue(attr string) (string, error) { - return defaultClient.ProjectAttributeValue(attr) -} - -// Scopes calls Client.Scopes on the default client. -func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) } - -func strsContains(ss []string, s string) bool { - for _, v := range ss { - if v == s { - return true - } - } - return false -} - -// A Client provides metadata. -type Client struct { - hc *http.Client -} - -// NewClient returns a Client that can be used to fetch metadata. -// Returns the client that uses the specified http.Client for HTTP requests. -// If nil is specified, returns the default client. -func NewClient(c *http.Client) *Client { - if c == nil { - return defaultClient - } - - return &Client{hc: c} -} - -// getETag returns a value from the metadata service as well as the associated ETag. -// This func is otherwise equivalent to Get. -func (c *Client) getETag(suffix string) (value, etag string, err error) { - ctx := context.TODO() - // Using a fixed IP makes it very difficult to spoof the metadata service in - // a container, which is an important use-case for local testing of cloud - // deployments. To enable spoofing of the metadata service, the environment - // variable GCE_METADATA_HOST is first inspected to decide where metadata - // requests shall go. - host := os.Getenv(metadataHostEnv) - if host == "" { - // Using 169.254.169.254 instead of "metadata" here because Go - // binaries built with the "netgo" tag and without cgo won't - // know the search suffix for "metadata" is - // ".google.internal", and this IP address is documented as - // being stable anyway. - host = metadataIP - } - suffix = strings.TrimLeft(suffix, "/") - u := "http://" + host + "/computeMetadata/v1/" + suffix - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return "", "", err - } - req.Header.Set("Metadata-Flavor", "Google") - req.Header.Set("User-Agent", userAgent) - var res *http.Response - var reqErr error - retryer := newRetryer() - for { - res, reqErr = c.hc.Do(req) - var code int - if res != nil { - code = res.StatusCode - } - if delay, shouldRetry := retryer.Retry(code, reqErr); shouldRetry { - if err := sleep(ctx, delay); err != nil { - return "", "", err - } - continue - } - break - } - if reqErr != nil { - return "", "", reqErr - } - defer res.Body.Close() - if res.StatusCode == http.StatusNotFound { - return "", "", NotDefinedError(suffix) - } - all, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", "", err - } - if res.StatusCode != 200 { - return "", "", &Error{Code: res.StatusCode, Message: string(all)} - } - return string(all), res.Header.Get("Etag"), nil -} - -// Get returns a value from the metadata service. -// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". -// -// If the GCE_METADATA_HOST environment variable is not defined, a default of -// 169.254.169.254 will be used instead. -// -// If the requested metadata is not defined, the returned error will -// be of type NotDefinedError. -func (c *Client) Get(suffix string) (string, error) { - val, _, err := c.getETag(suffix) - return val, err -} - -func (c *Client) getTrimmed(suffix string) (s string, err error) { - s, err = c.Get(suffix) - s = strings.TrimSpace(s) - return -} - -func (c *Client) lines(suffix string) ([]string, error) { - j, err := c.Get(suffix) - if err != nil { - return nil, err - } - s := strings.Split(strings.TrimSpace(j), "\n") - for i := range s { - s[i] = strings.TrimSpace(s[i]) - } - return s, nil -} - -// ProjectID returns the current instance's project ID string. -func (c *Client) ProjectID() (string, error) { return projID.get(c) } - -// NumericProjectID returns the current instance's numeric project ID. -func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) } - -// InstanceID returns the current VM's numeric instance ID. -func (c *Client) InstanceID() (string, error) { return instID.get(c) } - -// InternalIP returns the instance's primary internal IP address. -func (c *Client) InternalIP() (string, error) { - return c.getTrimmed("instance/network-interfaces/0/ip") -} - -// Email returns the email address associated with the service account. -// The account may be empty or the string "default" to use the instance's -// main account. -func (c *Client) Email(serviceAccount string) (string, error) { - if serviceAccount == "" { - serviceAccount = "default" - } - return c.getTrimmed("instance/service-accounts/" + serviceAccount + "/email") -} - -// ExternalIP returns the instance's primary external (public) IP address. -func (c *Client) ExternalIP() (string, error) { - return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") -} - -// Hostname returns the instance's hostname. This will be of the form -// ".c..internal". -func (c *Client) Hostname() (string, error) { - return c.getTrimmed("instance/hostname") -} - -// InstanceTags returns the list of user-defined instance tags, -// assigned when initially creating a GCE instance. -func (c *Client) InstanceTags() ([]string, error) { - var s []string - j, err := c.Get("instance/tags") - if err != nil { - return nil, err - } - if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil { - return nil, err - } - return s, nil -} - -// InstanceName returns the current VM's instance ID string. -func (c *Client) InstanceName() (string, error) { - return c.getTrimmed("instance/name") -} - -// Zone returns the current VM's zone, such as "us-central1-b". -func (c *Client) Zone() (string, error) { - zone, err := c.getTrimmed("instance/zone") - // zone is of the form "projects//zones/". - if err != nil { - return "", err - } - return zone[strings.LastIndex(zone, "/")+1:], nil -} - -// InstanceAttributes returns the list of user-defined attributes, -// assigned when initially creating a GCE VM instance. The value of an -// attribute can be obtained with InstanceAttributeValue. -func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") } - -// ProjectAttributes returns the list of user-defined attributes -// applying to the project as a whole, not just this VM. The value of -// an attribute can be obtained with ProjectAttributeValue. -func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") } - -// InstanceAttributeValue returns the value of the provided VM -// instance attribute. -// -// If the requested attribute is not defined, the returned error will -// be of type NotDefinedError. -// -// InstanceAttributeValue may return ("", nil) if the attribute was -// defined to be the empty string. -func (c *Client) InstanceAttributeValue(attr string) (string, error) { - return c.Get("instance/attributes/" + attr) -} - -// ProjectAttributeValue returns the value of the provided -// project attribute. -// -// If the requested attribute is not defined, the returned error will -// be of type NotDefinedError. -// -// ProjectAttributeValue may return ("", nil) if the attribute was -// defined to be the empty string. -func (c *Client) ProjectAttributeValue(attr string) (string, error) { - return c.Get("project/attributes/" + attr) -} - -// Scopes returns the service account scopes for the given account. -// The account may be empty or the string "default" to use the instance's -// main account. -func (c *Client) Scopes(serviceAccount string) ([]string, error) { - if serviceAccount == "" { - serviceAccount = "default" - } - return c.lines("instance/service-accounts/" + serviceAccount + "/scopes") -} - -// Subscribe subscribes to a value from the metadata service. -// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". -// The suffix may contain query parameters. -// -// Subscribe calls fn with the latest metadata value indicated by the provided -// suffix. If the metadata value is deleted, fn is called with the empty string -// and ok false. Subscribe blocks until fn returns a non-nil error or the value -// is deleted. Subscribe returns the error value returned from the last call to -// fn, which may be nil when ok == false. -func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error { - const failedSubscribeSleep = time.Second * 5 - - // First check to see if the metadata value exists at all. - val, lastETag, err := c.getETag(suffix) - if err != nil { - return err - } - - if err := fn(val, true); err != nil { - return err - } - - ok := true - if strings.ContainsRune(suffix, '?') { - suffix += "&wait_for_change=true&last_etag=" - } else { - suffix += "?wait_for_change=true&last_etag=" - } - for { - val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag)) - if err != nil { - if _, deleted := err.(NotDefinedError); !deleted { - time.Sleep(failedSubscribeSleep) - continue // Retry on other errors. - } - ok = false - } - lastETag = etag - - if err := fn(val, ok); err != nil || !ok { - return err - } - } -} - -// Error contains an error response from the server. -type Error struct { - // Code is the HTTP response status code. - Code int - // Message is the server response message. - Message string -} - -func (e *Error) Error() string { - return fmt.Sprintf("compute: Received %d `%s`", e.Code, e.Message) -} diff --git a/vendor/cloud.google.com/go/compute/metadata/retry.go b/vendor/cloud.google.com/go/compute/metadata/retry.go deleted file mode 100644 index 0f18f3cda..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/retry.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metadata - -import ( - "context" - "io" - "math/rand" - "net/http" - "time" -) - -const ( - maxRetryAttempts = 5 -) - -var ( - syscallRetryable = func(err error) bool { return false } -) - -// defaultBackoff is basically equivalent to gax.Backoff without the need for -// the dependency. -type defaultBackoff struct { - max time.Duration - mul float64 - cur time.Duration -} - -func (b *defaultBackoff) Pause() time.Duration { - d := time.Duration(1 + rand.Int63n(int64(b.cur))) - b.cur = time.Duration(float64(b.cur) * b.mul) - if b.cur > b.max { - b.cur = b.max - } - return d -} - -// sleep is the equivalent of gax.Sleep without the need for the dependency. -func sleep(ctx context.Context, d time.Duration) error { - t := time.NewTimer(d) - select { - case <-ctx.Done(): - t.Stop() - return ctx.Err() - case <-t.C: - return nil - } -} - -func newRetryer() *metadataRetryer { - return &metadataRetryer{bo: &defaultBackoff{ - cur: 100 * time.Millisecond, - max: 30 * time.Second, - mul: 2, - }} -} - -type backoff interface { - Pause() time.Duration -} - -type metadataRetryer struct { - bo backoff - attempts int -} - -func (r *metadataRetryer) Retry(status int, err error) (time.Duration, bool) { - if status == http.StatusOK { - return 0, false - } - retryOk := shouldRetry(status, err) - if !retryOk { - return 0, false - } - if r.attempts == maxRetryAttempts { - return 0, false - } - r.attempts++ - return r.bo.Pause(), true -} - -func shouldRetry(status int, err error) bool { - if 500 <= status && status <= 599 { - return true - } - if err == io.ErrUnexpectedEOF { - return true - } - // Transient network errors should be retried. - if syscallRetryable(err) { - return true - } - if err, ok := err.(interface{ Temporary() bool }); ok { - if err.Temporary() { - return true - } - } - if err, ok := err.(interface{ Unwrap() error }); ok { - return shouldRetry(status, err.Unwrap()) - } - return false -} diff --git a/vendor/cloud.google.com/go/compute/metadata/retry_linux.go b/vendor/cloud.google.com/go/compute/metadata/retry_linux.go deleted file mode 100644 index bb412f891..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/retry_linux.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build linux -// +build linux - -package metadata - -import "syscall" - -func init() { - // Initialize syscallRetryable to return true on transient socket-level - // errors. These errors are specific to Linux. - syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED } -} diff --git a/vendor/cloud.google.com/go/compute/metadata/tidyfix.go b/vendor/cloud.google.com/go/compute/metadata/tidyfix.go deleted file mode 100644 index 4cef48500..000000000 --- a/vendor/cloud.google.com/go/compute/metadata/tidyfix.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file, and the {{.RootMod}} import, won't actually become part of -// the resultant binary. -//go:build modhack -// +build modhack - -package metadata - -// Necessary for safely adding multi-module repo. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository -import _ "cloud.google.com/go/compute/internal" diff --git a/vendor/cloud.google.com/go/doc.go b/vendor/cloud.google.com/go/doc.go deleted file mode 100644 index 833878ec8..000000000 --- a/vendor/cloud.google.com/go/doc.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2014 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* -Package cloud is the root of the packages used to access Google Cloud -Services. See https://godoc.org/cloud.google.com/go for a full list -of sub-packages. - -# Client Options - -All clients in sub-packages are configurable via client options. These options are -described here: https://godoc.org/google.golang.org/api/option. - -## Endpoint Override - -Endpoint configuration is used to specify the URL to which requests are -sent. It is used for services that support or require regional endpoints, as well -as for other use cases such as [testing against fake -servers](https://github.com/googleapis/google-cloud-go/blob/main/testing.md#testing-grpc-services-using-fakes). - -For example, the Vertex AI service recommends that you configure the endpoint to the -location with the features you want that is closest to your physical location or the -location of your users. There is no global endpoint for Vertex AI. See -[Vertex AI - Locations](https://cloud.google.com/vertex-ai/docs/general/locations) -for more details. The following example demonstrates configuring a Vertex AI client -with a regional endpoint: - - ctx := context.Background() - endpoint := "us-central1-aiplatform.googleapis.com:443" - client, err := aiplatform.NewDatasetClient(ctx, option.WithEndpoint(endpoint)) - -# Authentication and Authorization - -All the clients in sub-packages support authentication via Google Application Default -Credentials (see https://cloud.google.com/docs/authentication/production), or -by providing a JSON key file for a Service Account. See examples below. - -Google Application Default Credentials (ADC) is the recommended way to authorize -and authenticate clients. For information on how to create and obtain -Application Default Credentials, see -https://cloud.google.com/docs/authentication/production. Here is an example -of a client using ADC to authenticate: - - client, err := secretmanager.NewClient(context.Background()) - if err != nil { - // TODO: handle error. - } - _ = client // Use the client. - -You can use a file with credentials to authenticate and authorize, such as a JSON -key file associated with a Google service account. Service Account keys can be -created and downloaded from -https://console.cloud.google.com/iam-admin/serviceaccounts. This example uses -the Secret Manger client, but the same steps apply to the other client libraries -underneath this package. Example: - - client, err := secretmanager.NewClient(context.Background(), - option.WithCredentialsFile("/path/to/service-account-key.json")) - if err != nil { - // TODO: handle error. - } - _ = client // Use the client. - -In some cases (for instance, you don't want to store secrets on disk), you can -create credentials from in-memory JSON and use the WithCredentials option. -The google package in this example is at golang.org/x/oauth2/google. -This example uses the Secret Manager client, but the same steps apply to -the other client libraries underneath this package. Note that scopes can be -found at https://developers.google.com/identity/protocols/oauth2/scopes, and -are also provided in all auto-generated libraries: for example, -cloud.google.com/go/secretmanager/apiv1 provides DefaultAuthScopes. Example: - - ctx := context.Background() - creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), secretmanager.DefaultAuthScopes()...) - if err != nil { - // TODO: handle error. - } - client, err := secretmanager.NewClient(ctx, option.WithCredentials(creds)) - if err != nil { - // TODO: handle error. - } - _ = client // Use the client. - -# Timeouts and Cancellation - -By default, non-streaming methods, like Create or Get, will have a default deadline applied to the -context provided at call time, unless a context deadline is already set. Streaming -methods have no default deadline and will run indefinitely. To set timeouts or -arrange for cancellation, use contexts. Transient -errors will be retried when correctness allows. - -Here is an example of how to set a timeout for an RPC, use context.WithTimeout: - - ctx := context.Background() - // Do not set a timeout on the context passed to NewClient: dialing happens - // asynchronously, and the context is used to refresh credentials in the - // background. - client, err := secretmanager.NewClient(ctx) - if err != nil { - // TODO: handle error. - } - // Time out if it takes more than 10 seconds to create a dataset. - tctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() // Always call cancel. - - req := &secretmanagerpb.DeleteSecretRequest{Name: "projects/project-id/secrets/name"} - if err := client.DeleteSecret(tctx, req); err != nil { - // TODO: handle error. - } - -Here is an example of how to arrange for an RPC to be canceled, use context.WithCancel: - - ctx := context.Background() - // Do not cancel the context passed to NewClient: dialing happens asynchronously, - // and the context is used to refresh credentials in the background. - client, err := secretmanager.NewClient(ctx) - if err != nil { - // TODO: handle error. - } - cctx, cancel := context.WithCancel(ctx) - defer cancel() // Always call cancel. - - // TODO: Make the cancel function available to whatever might want to cancel the - // call--perhaps a GUI button. - req := &secretmanagerpb.DeleteSecretRequest{Name: "projects/proj/secrets/name"} - if err := client.DeleteSecret(cctx, req); err != nil { - // TODO: handle error. - } - -To opt out of default deadlines, set the temporary environment variable -GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE to "true" prior to client -creation. This affects all Google Cloud Go client libraries. This opt-out -mechanism will be removed in a future release. File an issue at -https://github.com/googleapis/google-cloud-go if the default deadlines -cannot work for you. - -Do not attempt to control the initial connection (dialing) of a service by setting a -timeout on the context passed to NewClient. Dialing is non-blocking, so timeouts -would be ineffective and would only interfere with credential refreshing, which uses -the same context. - -# Connection Pooling - -Connection pooling differs in clients based on their transport. Cloud -clients either rely on HTTP or gRPC transports to communicate -with Google Cloud. - -Cloud clients that use HTTP (bigquery, compute, storage, and translate) rely on the -underlying HTTP transport to cache connections for later re-use. These are cached to -the default http.MaxIdleConns and http.MaxIdleConnsPerHost settings in -http.DefaultTransport. - -For gRPC clients (all others in this repo), connection pooling is configurable. Users -of cloud client libraries may specify option.WithGRPCConnectionPool(n) as a client -option to NewClient calls. This configures the underlying gRPC connections to be -pooled and addressed in a round robin fashion. - -# Using the Libraries with Docker - -Minimal docker images like Alpine lack CA certificates. This causes RPCs to appear to -hang, because gRPC retries indefinitely. See https://github.com/googleapis/google-cloud-go/issues/928 -for more information. - -# Debugging - -To see gRPC logs, set the environment variable GRPC_GO_LOG_SEVERITY_LEVEL. See -https://godoc.org/google.golang.org/grpc/grpclog for more information. - -For HTTP logging, set the GODEBUG environment variable to "http2debug=1" or "http2debug=2". - -# Inspecting errors - -Most of the errors returned by the generated clients are wrapped in an -[github.com/googleapis/gax-go/v2/apierror.APIError] and can be further unwrapped -into a [google.golang.org/grpc/status.Status] or -[google.golang.org/api/googleapi.Error] depending -on the transport used to make the call (gRPC or REST). Converting your errors to -these types can be a useful way to get more information about what went wrong -while debugging. - -[github.com/googleapis/gax-go/v2/apierror.APIError] gives access to specific -details in the error. The transport-specific errors can still be unwrapped using -the [github.com/googleapis/gax-go/v2/apierror.APIError]. - - if err != nil { - var ae *apierror.APIError - if errors.As(err, &ae) { - log.Println(ae.Reason()) - log.Println(ae.Details().Help.GetLinks()) - } - } - -If the gRPC transport was used, the [google.golang.org/grpc/status.Status] can -still be parsed using the [google.golang.org/grpc/status.FromError] function. - - if err != nil { - if s, ok := status.FromError(err); ok { - log.Println(s.Message()) - for _, d := range s.Proto().Details { - log.Println(d) - } - } - } - -If the REST transport was used, the [google.golang.org/api/googleapi.Error] can -be parsed in a similar way, allowing access to details such as the HTTP response -code. - - if err != nil { - var gerr *googleapi.Error - if errors.As(err, &gerr) { - log.Println(gerr.Message) - } - } - -# Client Stability - -Clients in this repository are considered alpha or beta unless otherwise -marked as stable in the README.md. Semver is not used to communicate stability -of clients. - -Alpha and beta clients may change or go away without notice. - -Clients marked stable will maintain compatibility with future versions for as -long as we can reasonably sustain. Incompatible changes might be made in some -situations, including: - -- Security bugs may prompt backwards-incompatible changes. - -- Situations in which components are no longer feasible to maintain without -making breaking changes, including removal. - -- Parts of the client surface may be outright unstable and subject to change. -These parts of the surface will be labeled with the note, "It is EXPERIMENTAL -and subject to change or removal without notice." -*/ -package cloud // import "cloud.google.com/go" diff --git a/vendor/cloud.google.com/go/iam/CHANGES.md b/vendor/cloud.google.com/go/iam/CHANGES.md deleted file mode 100644 index ced217827..000000000 --- a/vendor/cloud.google.com/go/iam/CHANGES.md +++ /dev/null @@ -1,62 +0,0 @@ -# Changes - -## [0.8.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.7.0...iam/v0.8.0) (2022-12-05) - - -### Features - -* **iam:** Start generating and refresh some libraries ([#7089](https://github.com/googleapis/google-cloud-go/issues/7089)) ([a9045ff](https://github.com/googleapis/google-cloud-go/commit/a9045ff191a711089c37f1d94a63522d9939ce38)) - -## [0.7.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.6.0...iam/v0.7.0) (2022-11-03) - - -### Features - -* **iam:** rewrite signatures in terms of new location ([3c4b2b3](https://github.com/googleapis/google-cloud-go/commit/3c4b2b34565795537aac1661e6af2442437e34ad)) - -## [0.6.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.5.0...iam/v0.6.0) (2022-10-25) - - -### Features - -* **iam:** start generating stubs dir ([de2d180](https://github.com/googleapis/google-cloud-go/commit/de2d18066dc613b72f6f8db93ca60146dabcfdcc)) - -## [0.5.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.4.0...iam/v0.5.0) (2022-09-28) - - -### Features - -* **iam:** remove ListApplicablePolicies ([52dddd1](https://github.com/googleapis/google-cloud-go/commit/52dddd1ed89fbe77e1859311c3b993a77a82bfc7)) - -## [0.4.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.3.0...iam/v0.4.0) (2022-09-06) - - -### Features - -* **iam:** start generating apiv2 ([#6605](https://github.com/googleapis/google-cloud-go/issues/6605)) ([a6004e7](https://github.com/googleapis/google-cloud-go/commit/a6004e762f782869cd85688937475744f7b17e50)) - -## [0.3.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.2.0...iam/v0.3.0) (2022-02-23) - - -### Features - -* **iam:** set versionClient to module version ([55f0d92](https://github.com/googleapis/google-cloud-go/commit/55f0d92bf112f14b024b4ab0076c9875a17423c9)) - -## [0.2.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.1.1...iam/v0.2.0) (2022-02-14) - - -### Features - -* **iam:** add file for tracking version ([17b36ea](https://github.com/googleapis/google-cloud-go/commit/17b36ead42a96b1a01105122074e65164357519e)) - -### [0.1.1](https://www.github.com/googleapis/google-cloud-go/compare/iam/v0.1.0...iam/v0.1.1) (2022-01-14) - - -### Bug Fixes - -* **iam:** run formatter ([#5277](https://www.github.com/googleapis/google-cloud-go/issues/5277)) ([8682e4e](https://www.github.com/googleapis/google-cloud-go/commit/8682e4ed57a4428a659fbc225f56c91767e2a4a9)) - -## v0.1.0 - -This is the first tag to carve out iam as its own module. See -[Add a module to a multi-module repository](https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository). diff --git a/vendor/cloud.google.com/go/iam/LICENSE b/vendor/cloud.google.com/go/iam/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/vendor/cloud.google.com/go/iam/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/cloud.google.com/go/iam/README.md b/vendor/cloud.google.com/go/iam/README.md deleted file mode 100644 index 0072cc9e2..000000000 --- a/vendor/cloud.google.com/go/iam/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# IAM API - -[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/iam.svg)](https://pkg.go.dev/cloud.google.com/go/iam) - -Go Client Library for IAM API. - -## Install - -```bash -go get cloud.google.com/go/iam -``` - -## Stability - -The stability of this module is indicated by SemVer. - -However, a `v1+` module may have breaking changes in two scenarios: - -* Packages with `alpha` or `beta` in the import path -* The GoDoc has an explicit stability disclaimer (for example, for an experimental feature). - -## Go Version Support - -See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) -section in the root directory's README. - -## Authorization - -See the [Authorization](https://github.com/googleapis/google-cloud-go#authorization) -section in the root directory's README. - -## Contributing - -Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) -document for details. - -Please note that this project is released with a Contributor Code of Conduct. -By participating in this project you agree to abide by its terms. See -[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) -for more information. diff --git a/vendor/cloud.google.com/go/iam/iam.go b/vendor/cloud.google.com/go/iam/iam.go deleted file mode 100644 index 0a06ea2e8..000000000 --- a/vendor/cloud.google.com/go/iam/iam.go +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright 2016 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package iam supports the resource-specific operations of Google Cloud -// IAM (Identity and Access Management) for the Google Cloud Libraries. -// See https://cloud.google.com/iam for more about IAM. -// -// Users of the Google Cloud Libraries will typically not use this package -// directly. Instead they will begin with some resource that supports IAM, like -// a pubsub topic, and call its IAM method to get a Handle for that resource. -package iam - -import ( - "context" - "fmt" - "time" - - gax "github.com/googleapis/gax-go/v2" - pb "google.golang.org/genproto/googleapis/iam/v1" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" -) - -// client abstracts the IAMPolicy API to allow multiple implementations. -type client interface { - Get(ctx context.Context, resource string) (*pb.Policy, error) - Set(ctx context.Context, resource string, p *pb.Policy) error - Test(ctx context.Context, resource string, perms []string) ([]string, error) - GetWithVersion(ctx context.Context, resource string, requestedPolicyVersion int32) (*pb.Policy, error) -} - -// grpcClient implements client for the standard gRPC-based IAMPolicy service. -type grpcClient struct { - c pb.IAMPolicyClient -} - -var withRetry = gax.WithRetry(func() gax.Retryer { - return gax.OnCodes([]codes.Code{ - codes.DeadlineExceeded, - codes.Unavailable, - }, gax.Backoff{ - Initial: 100 * time.Millisecond, - Max: 60 * time.Second, - Multiplier: 1.3, - }) -}) - -func (g *grpcClient) Get(ctx context.Context, resource string) (*pb.Policy, error) { - return g.GetWithVersion(ctx, resource, 1) -} - -func (g *grpcClient) GetWithVersion(ctx context.Context, resource string, requestedPolicyVersion int32) (*pb.Policy, error) { - var proto *pb.Policy - md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "resource", resource)) - ctx = insertMetadata(ctx, md) - - err := gax.Invoke(ctx, func(ctx context.Context, _ gax.CallSettings) error { - var err error - proto, err = g.c.GetIamPolicy(ctx, &pb.GetIamPolicyRequest{ - Resource: resource, - Options: &pb.GetPolicyOptions{ - RequestedPolicyVersion: requestedPolicyVersion, - }, - }) - return err - }, withRetry) - if err != nil { - return nil, err - } - return proto, nil -} - -func (g *grpcClient) Set(ctx context.Context, resource string, p *pb.Policy) error { - md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "resource", resource)) - ctx = insertMetadata(ctx, md) - - return gax.Invoke(ctx, func(ctx context.Context, _ gax.CallSettings) error { - _, err := g.c.SetIamPolicy(ctx, &pb.SetIamPolicyRequest{ - Resource: resource, - Policy: p, - }) - return err - }, withRetry) -} - -func (g *grpcClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) { - var res *pb.TestIamPermissionsResponse - md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "resource", resource)) - ctx = insertMetadata(ctx, md) - - err := gax.Invoke(ctx, func(ctx context.Context, _ gax.CallSettings) error { - var err error - res, err = g.c.TestIamPermissions(ctx, &pb.TestIamPermissionsRequest{ - Resource: resource, - Permissions: perms, - }) - return err - }, withRetry) - if err != nil { - return nil, err - } - return res.Permissions, nil -} - -// A Handle provides IAM operations for a resource. -type Handle struct { - c client - resource string -} - -// A Handle3 provides IAM operations for a resource. It is similar to a Handle, but provides access to newer IAM features (e.g., conditions). -type Handle3 struct { - c client - resource string - version int32 -} - -// InternalNewHandle is for use by the Google Cloud Libraries only. -// -// InternalNewHandle returns a Handle for resource. -// The conn parameter refers to a server that must support the IAMPolicy service. -func InternalNewHandle(conn grpc.ClientConnInterface, resource string) *Handle { - return InternalNewHandleGRPCClient(pb.NewIAMPolicyClient(conn), resource) -} - -// InternalNewHandleGRPCClient is for use by the Google Cloud Libraries only. -// -// InternalNewHandleClient returns a Handle for resource using the given -// grpc service that implements IAM as a mixin -func InternalNewHandleGRPCClient(c pb.IAMPolicyClient, resource string) *Handle { - return InternalNewHandleClient(&grpcClient{c: c}, resource) -} - -// InternalNewHandleClient is for use by the Google Cloud Libraries only. -// -// InternalNewHandleClient returns a Handle for resource using the given -// client implementation. -func InternalNewHandleClient(c client, resource string) *Handle { - return &Handle{ - c: c, - resource: resource, - } -} - -// V3 returns a Handle3, which is like Handle except it sets -// requestedPolicyVersion to 3 when retrieving a policy and policy.version to 3 -// when storing a policy. -func (h *Handle) V3() *Handle3 { - return &Handle3{ - c: h.c, - resource: h.resource, - version: 3, - } -} - -// Policy retrieves the IAM policy for the resource. -func (h *Handle) Policy(ctx context.Context) (*Policy, error) { - proto, err := h.c.Get(ctx, h.resource) - if err != nil { - return nil, err - } - return &Policy{InternalProto: proto}, nil -} - -// SetPolicy replaces the resource's current policy with the supplied Policy. -// -// If policy was created from a prior call to Get, then the modification will -// only succeed if the policy has not changed since the Get. -func (h *Handle) SetPolicy(ctx context.Context, policy *Policy) error { - return h.c.Set(ctx, h.resource, policy.InternalProto) -} - -// TestPermissions returns the subset of permissions that the caller has on the resource. -func (h *Handle) TestPermissions(ctx context.Context, permissions []string) ([]string, error) { - return h.c.Test(ctx, h.resource, permissions) -} - -// A RoleName is a name representing a collection of permissions. -type RoleName string - -// Common role names. -const ( - Owner RoleName = "roles/owner" - Editor RoleName = "roles/editor" - Viewer RoleName = "roles/viewer" -) - -const ( - // AllUsers is a special member that denotes all users, even unauthenticated ones. - AllUsers = "allUsers" - - // AllAuthenticatedUsers is a special member that denotes all authenticated users. - AllAuthenticatedUsers = "allAuthenticatedUsers" -) - -// A Policy is a list of Bindings representing roles -// granted to members. -// -// The zero Policy is a valid policy with no bindings. -type Policy struct { - // TODO(jba): when type aliases are available, put Policy into an internal package - // and provide an exported alias here. - - // This field is exported for use by the Google Cloud Libraries only. - // It may become unexported in a future release. - InternalProto *pb.Policy -} - -// Members returns the list of members with the supplied role. -// The return value should not be modified. Use Add and Remove -// to modify the members of a role. -func (p *Policy) Members(r RoleName) []string { - b := p.binding(r) - if b == nil { - return nil - } - return b.Members -} - -// HasRole reports whether member has role r. -func (p *Policy) HasRole(member string, r RoleName) bool { - return memberIndex(member, p.binding(r)) >= 0 -} - -// Add adds member member to role r if it is not already present. -// A new binding is created if there is no binding for the role. -func (p *Policy) Add(member string, r RoleName) { - b := p.binding(r) - if b == nil { - if p.InternalProto == nil { - p.InternalProto = &pb.Policy{} - } - p.InternalProto.Bindings = append(p.InternalProto.Bindings, &pb.Binding{ - Role: string(r), - Members: []string{member}, - }) - return - } - if memberIndex(member, b) < 0 { - b.Members = append(b.Members, member) - return - } -} - -// Remove removes member from role r if it is present. -func (p *Policy) Remove(member string, r RoleName) { - bi := p.bindingIndex(r) - if bi < 0 { - return - } - bindings := p.InternalProto.Bindings - b := bindings[bi] - mi := memberIndex(member, b) - if mi < 0 { - return - } - // Order doesn't matter for bindings or members, so to remove, move the last item - // into the removed spot and shrink the slice. - if len(b.Members) == 1 { - // Remove binding. - last := len(bindings) - 1 - bindings[bi] = bindings[last] - bindings[last] = nil - p.InternalProto.Bindings = bindings[:last] - return - } - // Remove member. - // TODO(jba): worry about multiple copies of m? - last := len(b.Members) - 1 - b.Members[mi] = b.Members[last] - b.Members[last] = "" - b.Members = b.Members[:last] -} - -// Roles returns the names of all the roles that appear in the Policy. -func (p *Policy) Roles() []RoleName { - if p.InternalProto == nil { - return nil - } - var rns []RoleName - for _, b := range p.InternalProto.Bindings { - rns = append(rns, RoleName(b.Role)) - } - return rns -} - -// binding returns the Binding for the suppied role, or nil if there isn't one. -func (p *Policy) binding(r RoleName) *pb.Binding { - i := p.bindingIndex(r) - if i < 0 { - return nil - } - return p.InternalProto.Bindings[i] -} - -func (p *Policy) bindingIndex(r RoleName) int { - if p.InternalProto == nil { - return -1 - } - for i, b := range p.InternalProto.Bindings { - if b.Role == string(r) { - return i - } - } - return -1 -} - -// memberIndex returns the index of m in b's Members, or -1 if not found. -func memberIndex(m string, b *pb.Binding) int { - if b == nil { - return -1 - } - for i, mm := range b.Members { - if mm == m { - return i - } - } - return -1 -} - -// insertMetadata inserts metadata into the given context -func insertMetadata(ctx context.Context, mds ...metadata.MD) context.Context { - out, _ := metadata.FromOutgoingContext(ctx) - out = out.Copy() - for _, md := range mds { - for k, v := range md { - out[k] = append(out[k], v...) - } - } - return metadata.NewOutgoingContext(ctx, out) -} - -// A Policy3 is a list of Bindings representing roles granted to members. -// -// The zero Policy3 is a valid policy with no bindings. -// -// It is similar to a Policy, except a Policy3 provides direct access to the -// list of Bindings. -// -// The policy version is always set to 3. -type Policy3 struct { - etag []byte - Bindings []*pb.Binding -} - -// Policy retrieves the IAM policy for the resource. -// -// requestedPolicyVersion is always set to 3. -func (h *Handle3) Policy(ctx context.Context) (*Policy3, error) { - proto, err := h.c.GetWithVersion(ctx, h.resource, h.version) - if err != nil { - return nil, err - } - return &Policy3{ - Bindings: proto.Bindings, - etag: proto.Etag, - }, nil -} - -// SetPolicy replaces the resource's current policy with the supplied Policy. -// -// If policy was created from a prior call to Get, then the modification will -// only succeed if the policy has not changed since the Get. -func (h *Handle3) SetPolicy(ctx context.Context, policy *Policy3) error { - return h.c.Set(ctx, h.resource, &pb.Policy{ - Bindings: policy.Bindings, - Etag: policy.etag, - Version: h.version, - }) -} - -// TestPermissions returns the subset of permissions that the caller has on the resource. -func (h *Handle3) TestPermissions(ctx context.Context, permissions []string) ([]string, error) { - return h.c.Test(ctx, h.resource, permissions) -} diff --git a/vendor/cloud.google.com/go/internal/.repo-metadata-full.json b/vendor/cloud.google.com/go/internal/.repo-metadata-full.json deleted file mode 100644 index 39bc0be8c..000000000 --- a/vendor/cloud.google.com/go/internal/.repo-metadata-full.json +++ /dev/null @@ -1,1946 +0,0 @@ -{ - "cloud.google.com/go/accessapproval/apiv1": { - "distribution_name": "cloud.google.com/go/accessapproval/apiv1", - "description": "Access Approval API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/accessapproval/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/accesscontextmanager/apiv1": { - "distribution_name": "cloud.google.com/go/accesscontextmanager/apiv1", - "description": "Access Context Manager API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/accesscontextmanager/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/aiplatform/apiv1": { - "distribution_name": "cloud.google.com/go/aiplatform/apiv1", - "description": "Vertex AI API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/aiplatform/apiv1beta1": { - "distribution_name": "cloud.google.com/go/aiplatform/apiv1beta1", - "description": "Vertex AI API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/analytics/admin/apiv1alpha": { - "distribution_name": "cloud.google.com/go/analytics/admin/apiv1alpha", - "description": "Google Analytics Admin API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/analytics/latest/admin/apiv1alpha", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/apigateway/apiv1": { - "distribution_name": "cloud.google.com/go/apigateway/apiv1", - "description": "API Gateway API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigateway/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/apigeeconnect/apiv1": { - "distribution_name": "cloud.google.com/go/apigeeconnect/apiv1", - "description": "Apigee Connect API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigeeconnect/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/apigeeregistry/apiv1": { - "distribution_name": "cloud.google.com/go/apigeeregistry/apiv1", - "description": "Apigee Registry API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigeeregistry/latest/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/apikeys/apiv2": { - "distribution_name": "cloud.google.com/go/apikeys/apiv2", - "description": "API Keys API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apikeys/latest/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/appengine/apiv1": { - "distribution_name": "cloud.google.com/go/appengine/apiv1", - "description": "App Engine Admin API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/appengine/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/area120/tables/apiv1alpha1": { - "distribution_name": "cloud.google.com/go/area120/tables/apiv1alpha1", - "description": "Area120 Tables API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/area120/latest/tables/apiv1alpha1", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/artifactregistry/apiv1": { - "distribution_name": "cloud.google.com/go/artifactregistry/apiv1", - "description": "Artifact Registry API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/artifactregistry/latest/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/artifactregistry/apiv1beta2": { - "distribution_name": "cloud.google.com/go/artifactregistry/apiv1beta2", - "description": "Artifact Registry API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/artifactregistry/latest/apiv1beta2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/asset/apiv1": { - "distribution_name": "cloud.google.com/go/asset/apiv1", - "description": "Cloud Asset API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/asset/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/asset/apiv1p2beta1": { - "distribution_name": "cloud.google.com/go/asset/apiv1p2beta1", - "description": "Cloud Asset API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/asset/latest/apiv1p2beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/asset/apiv1p5beta1": { - "distribution_name": "cloud.google.com/go/asset/apiv1p5beta1", - "description": "Cloud Asset API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/asset/latest/apiv1p5beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/assuredworkloads/apiv1": { - "distribution_name": "cloud.google.com/go/assuredworkloads/apiv1", - "description": "Assured Workloads API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/assuredworkloads/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/assuredworkloads/apiv1beta1": { - "distribution_name": "cloud.google.com/go/assuredworkloads/apiv1beta1", - "description": "Assured Workloads API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/assuredworkloads/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/automl/apiv1": { - "distribution_name": "cloud.google.com/go/automl/apiv1", - "description": "Cloud AutoML API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/automl/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/automl/apiv1beta1": { - "distribution_name": "cloud.google.com/go/automl/apiv1beta1", - "description": "Cloud AutoML API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/automl/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/baremetalsolution/apiv2": { - "distribution_name": "cloud.google.com/go/baremetalsolution/apiv2", - "description": "Bare Metal Solution API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/baremetalsolution/latest/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/batch/apiv1": { - "distribution_name": "cloud.google.com/go/batch/apiv1", - "description": "Batch API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/batch/latest/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/beyondcorp/appconnections/apiv1": { - "distribution_name": "cloud.google.com/go/beyondcorp/appconnections/apiv1", - "description": "BeyondCorp API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/appconnections/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/beyondcorp/appconnectors/apiv1": { - "distribution_name": "cloud.google.com/go/beyondcorp/appconnectors/apiv1", - "description": "BeyondCorp API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/appconnectors/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/beyondcorp/appgateways/apiv1": { - "distribution_name": "cloud.google.com/go/beyondcorp/appgateways/apiv1", - "description": "BeyondCorp API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/appgateways/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/beyondcorp/clientconnectorservices/apiv1": { - "distribution_name": "cloud.google.com/go/beyondcorp/clientconnectorservices/apiv1", - "description": "BeyondCorp API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/clientconnectorservices/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/beyondcorp/clientgateways/apiv1": { - "distribution_name": "cloud.google.com/go/beyondcorp/clientgateways/apiv1", - "description": "BeyondCorp API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/beyondcorp/latest/clientgateways/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery": { - "distribution_name": "cloud.google.com/go/bigquery", - "description": "BigQuery", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/bigquery/analyticshub/apiv1": { - "distribution_name": "cloud.google.com/go/bigquery/analyticshub/apiv1", - "description": "Analytics Hub API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/analyticshub/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/connection/apiv1": { - "distribution_name": "cloud.google.com/go/bigquery/connection/apiv1", - "description": "BigQuery Connection API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/connection/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/connection/apiv1beta1": { - "distribution_name": "cloud.google.com/go/bigquery/connection/apiv1beta1", - "description": "BigQuery Connection API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/connection/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/dataexchange/apiv1beta1": { - "distribution_name": "cloud.google.com/go/bigquery/dataexchange/apiv1beta1", - "description": "Analytics Hub API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/dataexchange/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/datapolicies/apiv1beta1": { - "distribution_name": "cloud.google.com/go/bigquery/datapolicies/apiv1beta1", - "description": "BigQuery Data Policy API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/datapolicies/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/datatransfer/apiv1": { - "distribution_name": "cloud.google.com/go/bigquery/datatransfer/apiv1", - "description": "BigQuery Data Transfer API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/datatransfer/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/migration/apiv2": { - "distribution_name": "cloud.google.com/go/bigquery/migration/apiv2", - "description": "BigQuery Migration API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/migration/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/migration/apiv2alpha": { - "distribution_name": "cloud.google.com/go/bigquery/migration/apiv2alpha", - "description": "BigQuery Migration API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/migration/apiv2alpha", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/reservation/apiv1": { - "distribution_name": "cloud.google.com/go/bigquery/reservation/apiv1", - "description": "BigQuery Reservation API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/reservation/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/reservation/apiv1beta1": { - "distribution_name": "cloud.google.com/go/bigquery/reservation/apiv1beta1", - "description": "BigQuery Reservation API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/reservation/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/storage/apiv1": { - "distribution_name": "cloud.google.com/go/bigquery/storage/apiv1", - "description": "BigQuery Storage API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/storage/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/storage/apiv1beta1": { - "distribution_name": "cloud.google.com/go/bigquery/storage/apiv1beta1", - "description": "BigQuery Storage API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/storage/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigquery/storage/apiv1beta2": { - "distribution_name": "cloud.google.com/go/bigquery/storage/apiv1beta2", - "description": "BigQuery Storage API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/storage/apiv1beta2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/bigtable": { - "distribution_name": "cloud.google.com/go/bigtable", - "description": "Cloud BigTable", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigtable/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/billing/apiv1": { - "distribution_name": "cloud.google.com/go/billing/apiv1", - "description": "Cloud Billing API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/billing/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/billing/budgets/apiv1": { - "distribution_name": "cloud.google.com/go/billing/budgets/apiv1", - "description": "Cloud Billing Budget API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/billing/latest/budgets/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/billing/budgets/apiv1beta1": { - "distribution_name": "cloud.google.com/go/billing/budgets/apiv1beta1", - "description": "Cloud Billing Budget API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/billing/latest/budgets/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/binaryauthorization/apiv1": { - "distribution_name": "cloud.google.com/go/binaryauthorization/apiv1", - "description": "Binary Authorization API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/binaryauthorization/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/binaryauthorization/apiv1beta1": { - "distribution_name": "cloud.google.com/go/binaryauthorization/apiv1beta1", - "description": "Binary Authorization API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/binaryauthorization/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/certificatemanager/apiv1": { - "distribution_name": "cloud.google.com/go/certificatemanager/apiv1", - "description": "Certificate Manager API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/certificatemanager/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/channel/apiv1": { - "distribution_name": "cloud.google.com/go/channel/apiv1", - "description": "Cloud Channel API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/channel/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/cloudbuild/apiv1/v2": { - "distribution_name": "cloud.google.com/go/cloudbuild/apiv1/v2", - "description": "Cloud Build API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudbuild/latest/apiv1/v2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/clouddms/apiv1": { - "distribution_name": "cloud.google.com/go/clouddms/apiv1", - "description": "Database Migration API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/clouddms/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/cloudtasks/apiv2": { - "distribution_name": "cloud.google.com/go/cloudtasks/apiv2", - "description": "Cloud Tasks API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudtasks/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/cloudtasks/apiv2beta2": { - "distribution_name": "cloud.google.com/go/cloudtasks/apiv2beta2", - "description": "Cloud Tasks API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudtasks/latest/apiv2beta2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/cloudtasks/apiv2beta3": { - "distribution_name": "cloud.google.com/go/cloudtasks/apiv2beta3", - "description": "Cloud Tasks API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudtasks/latest/apiv2beta3", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/compute/apiv1": { - "distribution_name": "cloud.google.com/go/compute/apiv1", - "description": "Google Compute Engine API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/compute/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/compute/metadata": { - "distribution_name": "cloud.google.com/go/compute/metadata", - "description": "Service Metadata API", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/compute/latest/metadata", - "release_level": "ga", - "library_type": "CORE" - }, - "cloud.google.com/go/contactcenterinsights/apiv1": { - "distribution_name": "cloud.google.com/go/contactcenterinsights/apiv1", - "description": "Contact Center AI Insights API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/contactcenterinsights/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/container/apiv1": { - "distribution_name": "cloud.google.com/go/container/apiv1", - "description": "Kubernetes Engine API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/container/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/containeranalysis/apiv1beta1": { - "distribution_name": "cloud.google.com/go/containeranalysis/apiv1beta1", - "description": "Container Analysis API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/containeranalysis/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datacatalog/apiv1": { - "distribution_name": "cloud.google.com/go/datacatalog/apiv1", - "description": "Google Cloud Data Catalog API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datacatalog/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datacatalog/apiv1beta1": { - "distribution_name": "cloud.google.com/go/datacatalog/apiv1beta1", - "description": "Google Cloud Data Catalog API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datacatalog/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dataflow/apiv1beta3": { - "distribution_name": "cloud.google.com/go/dataflow/apiv1beta3", - "description": "Dataflow API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataflow/latest/apiv1beta3", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dataform/apiv1alpha2": { - "distribution_name": "cloud.google.com/go/dataform/apiv1alpha2", - "description": "Dataform API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataform/latest/apiv1alpha2", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dataform/apiv1beta1": { - "distribution_name": "cloud.google.com/go/dataform/apiv1beta1", - "description": "Dataform API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataform/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datafusion/apiv1": { - "distribution_name": "cloud.google.com/go/datafusion/apiv1", - "description": "Cloud Data Fusion API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datafusion/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datalabeling/apiv1beta1": { - "distribution_name": "cloud.google.com/go/datalabeling/apiv1beta1", - "description": "Data Labeling API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datalabeling/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dataplex/apiv1": { - "distribution_name": "cloud.google.com/go/dataplex/apiv1", - "description": "Cloud Dataplex API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataplex/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dataproc/apiv1": { - "distribution_name": "cloud.google.com/go/dataproc/apiv1", - "description": "Cloud Dataproc API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataproc/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dataqna/apiv1alpha": { - "distribution_name": "cloud.google.com/go/dataqna/apiv1alpha", - "description": "Data QnA API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dataqna/latest/apiv1alpha", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datastore": { - "distribution_name": "cloud.google.com/go/datastore", - "description": "Cloud Datastore", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastore/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/datastore/admin/apiv1": { - "distribution_name": "cloud.google.com/go/datastore/admin/apiv1", - "description": "Cloud Datastore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastore/latest/admin/apiv1", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datastream/apiv1": { - "distribution_name": "cloud.google.com/go/datastream/apiv1", - "description": "Datastream API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastream/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/datastream/apiv1alpha1": { - "distribution_name": "cloud.google.com/go/datastream/apiv1alpha1", - "description": "Datastream API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/datastream/latest/apiv1alpha1", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/debugger/apiv2": { - "distribution_name": "cloud.google.com/go/debugger/apiv2", - "description": "Stackdriver Debugger API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/debugger/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/deploy/apiv1": { - "distribution_name": "cloud.google.com/go/deploy/apiv1", - "description": "Google Cloud Deploy API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/deploy/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dialogflow/apiv2": { - "distribution_name": "cloud.google.com/go/dialogflow/apiv2", - "description": "Dialogflow API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dialogflow/apiv2beta1": { - "distribution_name": "cloud.google.com/go/dialogflow/apiv2beta1", - "description": "Dialogflow API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/apiv2beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dialogflow/cx/apiv3": { - "distribution_name": "cloud.google.com/go/dialogflow/cx/apiv3", - "description": "Dialogflow API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/cx/apiv3", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dialogflow/cx/apiv3beta1": { - "distribution_name": "cloud.google.com/go/dialogflow/cx/apiv3beta1", - "description": "Dialogflow API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dialogflow/latest/cx/apiv3beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/dlp/apiv2": { - "distribution_name": "cloud.google.com/go/dlp/apiv2", - "description": "Cloud Data Loss Prevention (DLP) API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dlp/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/documentai/apiv1": { - "distribution_name": "cloud.google.com/go/documentai/apiv1", - "description": "Cloud Document AI API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/documentai/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/documentai/apiv1beta3": { - "distribution_name": "cloud.google.com/go/documentai/apiv1beta3", - "description": "Cloud Document AI API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/documentai/latest/apiv1beta3", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/domains/apiv1beta1": { - "distribution_name": "cloud.google.com/go/domains/apiv1beta1", - "description": "Cloud Domains API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/domains/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/edgecontainer/apiv1": { - "distribution_name": "cloud.google.com/go/edgecontainer/apiv1", - "description": "Distributed Cloud Edge Container API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/edgecontainer/latest/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/errorreporting": { - "distribution_name": "cloud.google.com/go/errorreporting", - "description": "Cloud Error Reporting API", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/errorreporting/latest", - "release_level": "beta", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/errorreporting/apiv1beta1": { - "distribution_name": "cloud.google.com/go/errorreporting/apiv1beta1", - "description": "Error Reporting API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/errorreporting/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/essentialcontacts/apiv1": { - "distribution_name": "cloud.google.com/go/essentialcontacts/apiv1", - "description": "Essential Contacts API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/essentialcontacts/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/eventarc/apiv1": { - "distribution_name": "cloud.google.com/go/eventarc/apiv1", - "description": "Eventarc API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/eventarc/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/eventarc/publishing/apiv1": { - "distribution_name": "cloud.google.com/go/eventarc/publishing/apiv1", - "description": "Eventarc Publishing API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/eventarc/latest/publishing/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/filestore/apiv1": { - "distribution_name": "cloud.google.com/go/filestore/apiv1", - "description": "Cloud Filestore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/filestore/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/firestore": { - "distribution_name": "cloud.google.com/go/firestore", - "description": "Cloud Firestore API", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/firestore/apiv1": { - "distribution_name": "cloud.google.com/go/firestore/apiv1", - "description": "Cloud Firestore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/firestore/apiv1/admin": { - "distribution_name": "cloud.google.com/go/firestore/apiv1/admin", - "description": "Cloud Firestore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest/apiv1/admin", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/functions/apiv1": { - "distribution_name": "cloud.google.com/go/functions/apiv1", - "description": "Cloud Functions API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/functions/apiv2": { - "distribution_name": "cloud.google.com/go/functions/apiv2", - "description": "Cloud Functions API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/functions/apiv2beta": { - "distribution_name": "cloud.google.com/go/functions/apiv2beta", - "description": "Cloud Functions API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/apiv2beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/functions/metadata": { - "distribution_name": "cloud.google.com/go/functions/metadata", - "description": "Cloud Functions", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/functions/latest/metadata", - "release_level": "alpha", - "library_type": "CORE" - }, - "cloud.google.com/go/gaming/apiv1": { - "distribution_name": "cloud.google.com/go/gaming/apiv1", - "description": "Game Services API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gaming/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/gaming/apiv1beta": { - "distribution_name": "cloud.google.com/go/gaming/apiv1beta", - "description": "Game Services API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gaming/latest/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/gkebackup/apiv1": { - "distribution_name": "cloud.google.com/go/gkebackup/apiv1", - "description": "Backup for GKE API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkebackup/latest/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/gkeconnect/gateway/apiv1beta1": { - "distribution_name": "cloud.google.com/go/gkeconnect/gateway/apiv1beta1", - "description": "Connect Gateway API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkeconnect/latest/gateway/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/gkehub/apiv1beta1": { - "distribution_name": "cloud.google.com/go/gkehub/apiv1beta1", - "description": "GKE Hub API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkehub/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/gkemulticloud/apiv1": { - "distribution_name": "cloud.google.com/go/gkemulticloud/apiv1", - "description": "Anthos Multi-Cloud API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gkemulticloud/latest/apiv1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/gsuiteaddons/apiv1": { - "distribution_name": "cloud.google.com/go/gsuiteaddons/apiv1", - "description": "Google Workspace Add-ons API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/gsuiteaddons/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/iam": { - "distribution_name": "cloud.google.com/go/iam", - "description": "Cloud IAM", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iam/latest", - "release_level": "ga", - "library_type": "CORE" - }, - "cloud.google.com/go/iam/apiv2": { - "distribution_name": "cloud.google.com/go/iam/apiv2", - "description": "Identity and Access Management (IAM) API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iam/latest/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/iam/credentials/apiv1": { - "distribution_name": "cloud.google.com/go/iam/credentials/apiv1", - "description": "IAM Service Account Credentials API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iam/latest/credentials/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/iap/apiv1": { - "distribution_name": "cloud.google.com/go/iap/apiv1", - "description": "Cloud Identity-Aware Proxy API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iap/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/ids/apiv1": { - "distribution_name": "cloud.google.com/go/ids/apiv1", - "description": "Cloud IDS API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/ids/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/iot/apiv1": { - "distribution_name": "cloud.google.com/go/iot/apiv1", - "description": "Cloud IoT API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/iot/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/kms/apiv1": { - "distribution_name": "cloud.google.com/go/kms/apiv1", - "description": "Cloud Key Management Service (KMS) API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/kms/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/language/apiv1": { - "distribution_name": "cloud.google.com/go/language/apiv1", - "description": "Cloud Natural Language API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/language/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/language/apiv1beta2": { - "distribution_name": "cloud.google.com/go/language/apiv1beta2", - "description": "Cloud Natural Language API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/language/latest/apiv1beta2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/lifesciences/apiv2beta": { - "distribution_name": "cloud.google.com/go/lifesciences/apiv2beta", - "description": "Cloud Life Sciences API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/lifesciences/latest/apiv2beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/logging": { - "distribution_name": "cloud.google.com/go/logging", - "description": "Cloud Logging API", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/logging/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/logging/apiv2": { - "distribution_name": "cloud.google.com/go/logging/apiv2", - "description": "Cloud Logging API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/logging/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/longrunning/autogen": { - "distribution_name": "cloud.google.com/go/longrunning/autogen", - "description": "Long Running Operations API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/longrunning/autogen", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/managedidentities/apiv1": { - "distribution_name": "cloud.google.com/go/managedidentities/apiv1", - "description": "Managed Service for Microsoft Active Directory API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/managedidentities/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/mediatranslation/apiv1beta1": { - "distribution_name": "cloud.google.com/go/mediatranslation/apiv1beta1", - "description": "Media Translation API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/mediatranslation/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/memcache/apiv1": { - "distribution_name": "cloud.google.com/go/memcache/apiv1", - "description": "Cloud Memorystore for Memcached API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/memcache/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/memcache/apiv1beta2": { - "distribution_name": "cloud.google.com/go/memcache/apiv1beta2", - "description": "Cloud Memorystore for Memcached API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/memcache/latest/apiv1beta2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/metastore/apiv1": { - "distribution_name": "cloud.google.com/go/metastore/apiv1", - "description": "Dataproc Metastore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/metastore/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/metastore/apiv1alpha": { - "distribution_name": "cloud.google.com/go/metastore/apiv1alpha", - "description": "Dataproc Metastore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/metastore/latest/apiv1alpha", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/metastore/apiv1beta": { - "distribution_name": "cloud.google.com/go/metastore/apiv1beta", - "description": "Dataproc Metastore API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/metastore/latest/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/monitoring/apiv3/v2": { - "distribution_name": "cloud.google.com/go/monitoring/apiv3/v2", - "description": "Cloud Monitoring API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/monitoring/latest/apiv3/v2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/monitoring/dashboard/apiv1": { - "distribution_name": "cloud.google.com/go/monitoring/dashboard/apiv1", - "description": "Cloud Monitoring API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/monitoring/latest/dashboard/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/monitoring/metricsscope/apiv1": { - "distribution_name": "cloud.google.com/go/monitoring/metricsscope/apiv1", - "description": "Cloud Monitoring API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/monitoring/latest/metricsscope/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/networkconnectivity/apiv1": { - "distribution_name": "cloud.google.com/go/networkconnectivity/apiv1", - "description": "Network Connectivity API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networkconnectivity/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/networkconnectivity/apiv1alpha1": { - "distribution_name": "cloud.google.com/go/networkconnectivity/apiv1alpha1", - "description": "Network Connectivity API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networkconnectivity/latest/apiv1alpha1", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/networkmanagement/apiv1": { - "distribution_name": "cloud.google.com/go/networkmanagement/apiv1", - "description": "Network Management API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networkmanagement/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/networksecurity/apiv1beta1": { - "distribution_name": "cloud.google.com/go/networksecurity/apiv1beta1", - "description": "Network Security API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/networksecurity/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/notebooks/apiv1": { - "distribution_name": "cloud.google.com/go/notebooks/apiv1", - "description": "Notebooks API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/notebooks/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/notebooks/apiv1beta1": { - "distribution_name": "cloud.google.com/go/notebooks/apiv1beta1", - "description": "Notebooks API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/notebooks/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/optimization/apiv1": { - "distribution_name": "cloud.google.com/go/optimization/apiv1", - "description": "Cloud Optimization API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/optimization/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/orchestration/airflow/service/apiv1": { - "distribution_name": "cloud.google.com/go/orchestration/airflow/service/apiv1", - "description": "Cloud Composer API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/orchestration/latest/airflow/service/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/orgpolicy/apiv2": { - "distribution_name": "cloud.google.com/go/orgpolicy/apiv2", - "description": "Organization Policy API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/orgpolicy/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/osconfig/agentendpoint/apiv1": { - "distribution_name": "cloud.google.com/go/osconfig/agentendpoint/apiv1", - "description": "OS Config API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/agentendpoint/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/osconfig/agentendpoint/apiv1beta": { - "distribution_name": "cloud.google.com/go/osconfig/agentendpoint/apiv1beta", - "description": "OS Config API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/agentendpoint/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/osconfig/apiv1": { - "distribution_name": "cloud.google.com/go/osconfig/apiv1", - "description": "OS Config API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/osconfig/apiv1alpha": { - "distribution_name": "cloud.google.com/go/osconfig/apiv1alpha", - "description": "OS Config API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/apiv1alpha", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/osconfig/apiv1beta": { - "distribution_name": "cloud.google.com/go/osconfig/apiv1beta", - "description": "OS Config API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/osconfig/latest/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/oslogin/apiv1": { - "distribution_name": "cloud.google.com/go/oslogin/apiv1", - "description": "Cloud OS Login API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/oslogin/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/oslogin/apiv1beta": { - "distribution_name": "cloud.google.com/go/oslogin/apiv1beta", - "description": "Cloud OS Login API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/oslogin/latest/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/phishingprotection/apiv1beta1": { - "distribution_name": "cloud.google.com/go/phishingprotection/apiv1beta1", - "description": "Phishing Protection API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/phishingprotection/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/policytroubleshooter/apiv1": { - "distribution_name": "cloud.google.com/go/policytroubleshooter/apiv1", - "description": "Policy Troubleshooter API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/policytroubleshooter/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/privatecatalog/apiv1beta1": { - "distribution_name": "cloud.google.com/go/privatecatalog/apiv1beta1", - "description": "Cloud Private Catalog API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/privatecatalog/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/profiler": { - "distribution_name": "cloud.google.com/go/profiler", - "description": "Cloud Profiler", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/profiler/latest", - "release_level": "ga", - "library_type": "AGENT" - }, - "cloud.google.com/go/pubsub": { - "distribution_name": "cloud.google.com/go/pubsub", - "description": "Cloud PubSub", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsub/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/pubsub/apiv1": { - "distribution_name": "cloud.google.com/go/pubsub/apiv1", - "description": "Cloud Pub/Sub API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsub/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/pubsublite": { - "distribution_name": "cloud.google.com/go/pubsublite", - "description": "Cloud PubSub Lite", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsublite/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/pubsublite/apiv1": { - "distribution_name": "cloud.google.com/go/pubsublite/apiv1", - "description": "Pub/Sub Lite API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/pubsublite/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/recaptchaenterprise/v2/apiv1": { - "distribution_name": "cloud.google.com/go/recaptchaenterprise/v2/apiv1", - "description": "reCAPTCHA Enterprise API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recaptchaenterprise/v2/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1": { - "distribution_name": "cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1", - "description": "reCAPTCHA Enterprise API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recaptchaenterprise/v2/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/recommendationengine/apiv1beta1": { - "distribution_name": "cloud.google.com/go/recommendationengine/apiv1beta1", - "description": "Recommendations AI", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recommendationengine/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/recommender/apiv1": { - "distribution_name": "cloud.google.com/go/recommender/apiv1", - "description": "Recommender API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recommender/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/recommender/apiv1beta1": { - "distribution_name": "cloud.google.com/go/recommender/apiv1beta1", - "description": "Recommender API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/recommender/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/redis/apiv1": { - "distribution_name": "cloud.google.com/go/redis/apiv1", - "description": "Google Cloud Memorystore for Redis API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/redis/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/redis/apiv1beta1": { - "distribution_name": "cloud.google.com/go/redis/apiv1beta1", - "description": "Google Cloud Memorystore for Redis API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/redis/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/resourcemanager/apiv2": { - "distribution_name": "cloud.google.com/go/resourcemanager/apiv2", - "description": "Cloud Resource Manager API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/resourcemanager/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/resourcemanager/apiv3": { - "distribution_name": "cloud.google.com/go/resourcemanager/apiv3", - "description": "Cloud Resource Manager API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/resourcemanager/latest/apiv3", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/resourcesettings/apiv1": { - "distribution_name": "cloud.google.com/go/resourcesettings/apiv1", - "description": "Resource Settings API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/resourcesettings/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/retail/apiv2": { - "distribution_name": "cloud.google.com/go/retail/apiv2", - "description": "Retail API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/retail/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/retail/apiv2alpha": { - "distribution_name": "cloud.google.com/go/retail/apiv2alpha", - "description": "Retail API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/retail/latest/apiv2alpha", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/retail/apiv2beta": { - "distribution_name": "cloud.google.com/go/retail/apiv2beta", - "description": "Retail API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/retail/latest/apiv2beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/rpcreplay": { - "distribution_name": "cloud.google.com/go/rpcreplay", - "description": "RPC Replay", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/rpcreplay", - "release_level": "ga", - "library_type": "OTHER" - }, - "cloud.google.com/go/run/apiv2": { - "distribution_name": "cloud.google.com/go/run/apiv2", - "description": "Cloud Run Admin API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/run/latest/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/scheduler/apiv1": { - "distribution_name": "cloud.google.com/go/scheduler/apiv1", - "description": "Cloud Scheduler API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/scheduler/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/scheduler/apiv1beta1": { - "distribution_name": "cloud.google.com/go/scheduler/apiv1beta1", - "description": "Cloud Scheduler API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/scheduler/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/secretmanager/apiv1": { - "distribution_name": "cloud.google.com/go/secretmanager/apiv1", - "description": "Secret Manager API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/secretmanager/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/security/privateca/apiv1": { - "distribution_name": "cloud.google.com/go/security/privateca/apiv1", - "description": "Certificate Authority API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/security/latest/privateca/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/security/privateca/apiv1beta1": { - "distribution_name": "cloud.google.com/go/security/privateca/apiv1beta1", - "description": "Certificate Authority API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/security/latest/privateca/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/security/publicca/apiv1beta1": { - "distribution_name": "cloud.google.com/go/security/publicca/apiv1beta1", - "description": "Public Certificate Authority API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/security/latest/publicca/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/securitycenter/apiv1": { - "distribution_name": "cloud.google.com/go/securitycenter/apiv1", - "description": "Security Command Center API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/securitycenter/apiv1beta1": { - "distribution_name": "cloud.google.com/go/securitycenter/apiv1beta1", - "description": "Security Command Center API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/securitycenter/apiv1p1beta1": { - "distribution_name": "cloud.google.com/go/securitycenter/apiv1p1beta1", - "description": "Security Command Center API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv1p1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/securitycenter/settings/apiv1beta1": { - "distribution_name": "cloud.google.com/go/securitycenter/settings/apiv1beta1", - "description": "Cloud Security Command Center API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/settings/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/servicecontrol/apiv1": { - "distribution_name": "cloud.google.com/go/servicecontrol/apiv1", - "description": "Service Control API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicecontrol/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/servicedirectory/apiv1": { - "distribution_name": "cloud.google.com/go/servicedirectory/apiv1", - "description": "Service Directory API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicedirectory/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/servicedirectory/apiv1beta1": { - "distribution_name": "cloud.google.com/go/servicedirectory/apiv1beta1", - "description": "Service Directory API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicedirectory/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/servicemanagement/apiv1": { - "distribution_name": "cloud.google.com/go/servicemanagement/apiv1", - "description": "Service Management API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicemanagement/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/serviceusage/apiv1": { - "distribution_name": "cloud.google.com/go/serviceusage/apiv1", - "description": "Service Usage API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/serviceusage/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/shell/apiv1": { - "distribution_name": "cloud.google.com/go/shell/apiv1", - "description": "Cloud Shell API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/shell/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/spanner": { - "distribution_name": "cloud.google.com/go/spanner", - "description": "Cloud Spanner", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/spanner/admin/database/apiv1": { - "distribution_name": "cloud.google.com/go/spanner/admin/database/apiv1", - "description": "Cloud Spanner API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/admin/database/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/spanner/admin/instance/apiv1": { - "distribution_name": "cloud.google.com/go/spanner/admin/instance/apiv1", - "description": "Cloud Spanner Instance Admin API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/admin/instance/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/spanner/apiv1": { - "distribution_name": "cloud.google.com/go/spanner/apiv1", - "description": "Cloud Spanner API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/speech/apiv1": { - "distribution_name": "cloud.google.com/go/speech/apiv1", - "description": "Cloud Speech-to-Text API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/speech/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/speech/apiv1p1beta1": { - "distribution_name": "cloud.google.com/go/speech/apiv1p1beta1", - "description": "Cloud Speech-to-Text API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/speech/latest/apiv1p1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/speech/apiv2": { - "distribution_name": "cloud.google.com/go/speech/apiv2", - "description": "Cloud Speech-to-Text API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/speech/latest/apiv2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/storage": { - "distribution_name": "cloud.google.com/go/storage", - "description": "Cloud Storage (GCS)", - "language": "Go", - "client_library_type": "manual", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest", - "release_level": "ga", - "library_type": "GAPIC_MANUAL" - }, - "cloud.google.com/go/storage/internal/apiv2": { - "distribution_name": "cloud.google.com/go/storage/internal/apiv2", - "description": "Cloud Storage API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest/internal/apiv2", - "release_level": "alpha", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/storagetransfer/apiv1": { - "distribution_name": "cloud.google.com/go/storagetransfer/apiv1", - "description": "Storage Transfer API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storagetransfer/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/talent/apiv4": { - "distribution_name": "cloud.google.com/go/talent/apiv4", - "description": "Cloud Talent Solution API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/talent/latest/apiv4", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/talent/apiv4beta1": { - "distribution_name": "cloud.google.com/go/talent/apiv4beta1", - "description": "Cloud Talent Solution API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/talent/latest/apiv4beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/texttospeech/apiv1": { - "distribution_name": "cloud.google.com/go/texttospeech/apiv1", - "description": "Cloud Text-to-Speech API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/texttospeech/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/tpu/apiv1": { - "distribution_name": "cloud.google.com/go/tpu/apiv1", - "description": "Cloud TPU API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/tpu/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/trace/apiv1": { - "distribution_name": "cloud.google.com/go/trace/apiv1", - "description": "Stackdriver Trace API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/trace/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/trace/apiv2": { - "distribution_name": "cloud.google.com/go/trace/apiv2", - "description": "Stackdriver Trace API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/trace/latest/apiv2", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/translate/apiv3": { - "distribution_name": "cloud.google.com/go/translate/apiv3", - "description": "Cloud Translation API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/translate/latest/apiv3", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/video/livestream/apiv1": { - "distribution_name": "cloud.google.com/go/video/livestream/apiv1", - "description": "Live Stream API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/livestream/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/video/stitcher/apiv1": { - "distribution_name": "cloud.google.com/go/video/stitcher/apiv1", - "description": "Video Stitcher API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/stitcher/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/video/transcoder/apiv1": { - "distribution_name": "cloud.google.com/go/video/transcoder/apiv1", - "description": "Transcoder API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/transcoder/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/videointelligence/apiv1": { - "distribution_name": "cloud.google.com/go/videointelligence/apiv1", - "description": "Cloud Video Intelligence API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/videointelligence/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/videointelligence/apiv1beta2": { - "distribution_name": "cloud.google.com/go/videointelligence/apiv1beta2", - "description": "Google Cloud Video Intelligence API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/videointelligence/latest/apiv1beta2", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/videointelligence/apiv1p3beta1": { - "distribution_name": "cloud.google.com/go/videointelligence/apiv1p3beta1", - "description": "Cloud Video Intelligence API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/videointelligence/latest/apiv1p3beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/vision/v2/apiv1": { - "distribution_name": "cloud.google.com/go/vision/v2/apiv1", - "description": "Cloud Vision API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vision/v2/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/vision/v2/apiv1p1beta1": { - "distribution_name": "cloud.google.com/go/vision/v2/apiv1p1beta1", - "description": "Cloud Vision API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vision/v2/latest/apiv1p1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/vmmigration/apiv1": { - "distribution_name": "cloud.google.com/go/vmmigration/apiv1", - "description": "VM Migration API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vmmigration/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/vpcaccess/apiv1": { - "distribution_name": "cloud.google.com/go/vpcaccess/apiv1", - "description": "Serverless VPC Access API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/vpcaccess/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/webrisk/apiv1": { - "distribution_name": "cloud.google.com/go/webrisk/apiv1", - "description": "Web Risk API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/webrisk/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/webrisk/apiv1beta1": { - "distribution_name": "cloud.google.com/go/webrisk/apiv1beta1", - "description": "Web Risk API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/webrisk/latest/apiv1beta1", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/websecurityscanner/apiv1": { - "distribution_name": "cloud.google.com/go/websecurityscanner/apiv1", - "description": "Web Security Scanner API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/websecurityscanner/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/workflows/apiv1": { - "distribution_name": "cloud.google.com/go/workflows/apiv1", - "description": "Workflows API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/workflows/apiv1beta": { - "distribution_name": "cloud.google.com/go/workflows/apiv1beta", - "description": "Workflows API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/workflows/executions/apiv1": { - "distribution_name": "cloud.google.com/go/workflows/executions/apiv1", - "description": "Workflow Executions API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/executions/apiv1", - "release_level": "ga", - "library_type": "GAPIC_AUTO" - }, - "cloud.google.com/go/workflows/executions/apiv1beta": { - "distribution_name": "cloud.google.com/go/workflows/executions/apiv1beta", - "description": "Workflow Executions API", - "language": "Go", - "client_library_type": "generated", - "docs_url": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/workflows/latest/executions/apiv1beta", - "release_level": "beta", - "library_type": "GAPIC_AUTO" - } -} diff --git a/vendor/cloud.google.com/go/internal/README.md b/vendor/cloud.google.com/go/internal/README.md deleted file mode 100644 index 8857c8f6f..000000000 --- a/vendor/cloud.google.com/go/internal/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Internal - -This directory contains internal code for cloud.google.com/go packages. - -## .repo-metadata-full.json - -`.repo-metadata-full.json` contains metadata about the packages in this repo. It -is generated by `internal/gapicgen/generator`. It's processed by external tools -to build lists of all of the packages. - -Don't make breaking changes to the format without consulting with the external -tools. - -One day, we may want to create individual `.repo-metadata.json` files next to -each package, which is the pattern followed by some other languages. External -tools would then talk to pkg.go.dev or some other service to get the overall -list of packages and use the `.repo-metadata.json` files to get the additional -metadata required. For now, `.repo-metadata-full.json` includes everything. \ No newline at end of file diff --git a/vendor/cloud.google.com/go/internal/annotate.go b/vendor/cloud.google.com/go/internal/annotate.go deleted file mode 100644 index 30d7bcf77..000000000 --- a/vendor/cloud.google.com/go/internal/annotate.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "fmt" - - "google.golang.org/api/googleapi" - "google.golang.org/grpc/status" -) - -// Annotate prepends msg to the error message in err, attempting -// to preserve other information in err, like an error code. -// -// Annotate panics if err is nil. -// -// Annotate knows about these error types: -// - "google.golang.org/grpc/status".Status -// - "google.golang.org/api/googleapi".Error -// If the error is not one of these types, Annotate behaves -// like -// -// fmt.Errorf("%s: %v", msg, err) -func Annotate(err error, msg string) error { - if err == nil { - panic("Annotate called with nil") - } - if s, ok := status.FromError(err); ok { - p := s.Proto() - p.Message = msg + ": " + p.Message - return status.ErrorProto(p) - } - if g, ok := err.(*googleapi.Error); ok { - g.Message = msg + ": " + g.Message - return g - } - return fmt.Errorf("%s: %v", msg, err) -} - -// Annotatef uses format and args to format a string, then calls Annotate. -func Annotatef(err error, format string, args ...interface{}) error { - return Annotate(err, fmt.Sprintf(format, args...)) -} diff --git a/vendor/cloud.google.com/go/internal/optional/optional.go b/vendor/cloud.google.com/go/internal/optional/optional.go deleted file mode 100644 index 72780f764..000000000 --- a/vendor/cloud.google.com/go/internal/optional/optional.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2016 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package optional provides versions of primitive types that can -// be nil. These are useful in methods that update some of an API object's -// fields. -package optional - -import ( - "fmt" - "strings" - "time" -) - -type ( - // Bool is either a bool or nil. - Bool interface{} - - // String is either a string or nil. - String interface{} - - // Int is either an int or nil. - Int interface{} - - // Uint is either a uint or nil. - Uint interface{} - - // Float64 is either a float64 or nil. - Float64 interface{} - - // Duration is either a time.Duration or nil. - Duration interface{} -) - -// ToBool returns its argument as a bool. -// It panics if its argument is nil or not a bool. -func ToBool(v Bool) bool { - x, ok := v.(bool) - if !ok { - doPanic("Bool", v) - } - return x -} - -// ToString returns its argument as a string. -// It panics if its argument is nil or not a string. -func ToString(v String) string { - x, ok := v.(string) - if !ok { - doPanic("String", v) - } - return x -} - -// ToInt returns its argument as an int. -// It panics if its argument is nil or not an int. -func ToInt(v Int) int { - x, ok := v.(int) - if !ok { - doPanic("Int", v) - } - return x -} - -// ToUint returns its argument as a uint. -// It panics if its argument is nil or not a uint. -func ToUint(v Uint) uint { - x, ok := v.(uint) - if !ok { - doPanic("Uint", v) - } - return x -} - -// ToFloat64 returns its argument as a float64. -// It panics if its argument is nil or not a float64. -func ToFloat64(v Float64) float64 { - x, ok := v.(float64) - if !ok { - doPanic("Float64", v) - } - return x -} - -// ToDuration returns its argument as a time.Duration. -// It panics if its argument is nil or not a time.Duration. -func ToDuration(v Duration) time.Duration { - x, ok := v.(time.Duration) - if !ok { - doPanic("Duration", v) - } - return x -} - -func doPanic(capType string, v interface{}) { - panic(fmt.Sprintf("optional.%s value should be %s, got %T", capType, strings.ToLower(capType), v)) -} diff --git a/vendor/cloud.google.com/go/internal/retry.go b/vendor/cloud.google.com/go/internal/retry.go deleted file mode 100644 index 2943a6d0b..000000000 --- a/vendor/cloud.google.com/go/internal/retry.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2016 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "context" - "fmt" - "time" - - gax "github.com/googleapis/gax-go/v2" - "google.golang.org/grpc/status" -) - -// Retry calls the supplied function f repeatedly according to the provided -// backoff parameters. It returns when one of the following occurs: -// When f's first return value is true, Retry immediately returns with f's second -// return value. -// When the provided context is done, Retry returns with an error that -// includes both ctx.Error() and the last error returned by f. -func Retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error)) error { - return retry(ctx, bo, f, gax.Sleep) -} - -func retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error), - sleep func(context.Context, time.Duration) error) error { - var lastErr error - for { - stop, err := f() - if stop { - return err - } - // Remember the last "real" error from f. - if err != nil && err != context.Canceled && err != context.DeadlineExceeded { - lastErr = err - } - p := bo.Pause() - if ctxErr := sleep(ctx, p); ctxErr != nil { - if lastErr != nil { - return wrappedCallErr{ctxErr: ctxErr, wrappedErr: lastErr} - } - return ctxErr - } - } -} - -// Use this error type to return an error which allows introspection of both -// the context error and the error from the service. -type wrappedCallErr struct { - ctxErr error - wrappedErr error -} - -func (e wrappedCallErr) Error() string { - return fmt.Sprintf("retry failed with %v; last error: %v", e.ctxErr, e.wrappedErr) -} - -func (e wrappedCallErr) Unwrap() error { - return e.wrappedErr -} - -// Is allows errors.Is to match the error from the call as well as context -// sentinel errors. -func (e wrappedCallErr) Is(err error) bool { - return e.ctxErr == err || e.wrappedErr == err -} - -// GRPCStatus allows the wrapped error to be used with status.FromError. -func (e wrappedCallErr) GRPCStatus() *status.Status { - if s, ok := status.FromError(e.wrappedErr); ok { - return s - } - return nil -} diff --git a/vendor/cloud.google.com/go/internal/trace/trace.go b/vendor/cloud.google.com/go/internal/trace/trace.go deleted file mode 100644 index c201d343e..000000000 --- a/vendor/cloud.google.com/go/internal/trace/trace.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package trace - -import ( - "context" - "fmt" - - "go.opencensus.io/trace" - "golang.org/x/xerrors" - "google.golang.org/api/googleapi" - "google.golang.org/genproto/googleapis/rpc/code" - "google.golang.org/grpc/status" -) - -// StartSpan adds a span to the trace with the given name. -func StartSpan(ctx context.Context, name string) context.Context { - ctx, _ = trace.StartSpan(ctx, name) - return ctx -} - -// EndSpan ends a span with the given error. -func EndSpan(ctx context.Context, err error) { - span := trace.FromContext(ctx) - if err != nil { - span.SetStatus(toStatus(err)) - } - span.End() -} - -// toStatus interrogates an error and converts it to an appropriate -// OpenCensus status. -func toStatus(err error) trace.Status { - var err2 *googleapi.Error - if ok := xerrors.As(err, &err2); ok { - return trace.Status{Code: httpStatusCodeToOCCode(err2.Code), Message: err2.Message} - } else if s, ok := status.FromError(err); ok { - return trace.Status{Code: int32(s.Code()), Message: s.Message()} - } else { - return trace.Status{Code: int32(code.Code_UNKNOWN), Message: err.Error()} - } -} - -// TODO(deklerk): switch to using OpenCensus function when it becomes available. -// Reference: https://github.com/googleapis/googleapis/blob/26b634d2724ac5dd30ae0b0cbfb01f07f2e4050e/google/rpc/code.proto -func httpStatusCodeToOCCode(httpStatusCode int) int32 { - switch httpStatusCode { - case 200: - return int32(code.Code_OK) - case 499: - return int32(code.Code_CANCELLED) - case 500: - return int32(code.Code_UNKNOWN) // Could also be Code_INTERNAL, Code_DATA_LOSS - case 400: - return int32(code.Code_INVALID_ARGUMENT) // Could also be Code_OUT_OF_RANGE - case 504: - return int32(code.Code_DEADLINE_EXCEEDED) - case 404: - return int32(code.Code_NOT_FOUND) - case 409: - return int32(code.Code_ALREADY_EXISTS) // Could also be Code_ABORTED - case 403: - return int32(code.Code_PERMISSION_DENIED) - case 401: - return int32(code.Code_UNAUTHENTICATED) - case 429: - return int32(code.Code_RESOURCE_EXHAUSTED) - case 501: - return int32(code.Code_UNIMPLEMENTED) - case 503: - return int32(code.Code_UNAVAILABLE) - default: - return int32(code.Code_UNKNOWN) - } -} - -// TODO: (odeke-em): perhaps just pass around spans due to the cost -// incurred from using trace.FromContext(ctx) yet we could avoid -// throwing away the work done by ctx, span := trace.StartSpan. -func TracePrintf(ctx context.Context, attrMap map[string]interface{}, format string, args ...interface{}) { - var attrs []trace.Attribute - for k, v := range attrMap { - var a trace.Attribute - switch v := v.(type) { - case string: - a = trace.StringAttribute(k, v) - case bool: - a = trace.BoolAttribute(k, v) - case int: - a = trace.Int64Attribute(k, int64(v)) - case int64: - a = trace.Int64Attribute(k, v) - default: - a = trace.StringAttribute(k, fmt.Sprintf("%#v", v)) - } - attrs = append(attrs, a) - } - trace.FromContext(ctx).Annotatef(attrs, format, args...) -} diff --git a/vendor/cloud.google.com/go/internal/version/update_version.sh b/vendor/cloud.google.com/go/internal/version/update_version.sh deleted file mode 100644 index d7c5a3e21..000000000 --- a/vendor/cloud.google.com/go/internal/version/update_version.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -today=$(date +%Y%m%d) - -sed -i -r -e 's/const Repo = "([0-9]{8})"/const Repo = "'$today'"/' $GOFILE - diff --git a/vendor/cloud.google.com/go/internal/version/version.go b/vendor/cloud.google.com/go/internal/version/version.go deleted file mode 100644 index fd9dd91e9..000000000 --- a/vendor/cloud.google.com/go/internal/version/version.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2016 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:generate ./update_version.sh - -// Package version contains version information for Google Cloud Client -// Libraries for Go, as reported in request headers. -package version - -import ( - "runtime" - "strings" - "unicode" -) - -// Repo is the current version of the client libraries in this -// repo. It should be a date in YYYYMMDD format. -const Repo = "20201104" - -// Go returns the Go runtime version. The returned string -// has no whitespace. -func Go() string { - return goVersion -} - -var goVersion = goVer(runtime.Version()) - -const develPrefix = "devel +" - -func goVer(s string) string { - if strings.HasPrefix(s, develPrefix) { - s = s[len(develPrefix):] - if p := strings.IndexFunc(s, unicode.IsSpace); p >= 0 { - s = s[:p] - } - return s - } - - if strings.HasPrefix(s, "go1") { - s = s[2:] - var prerelease string - if p := strings.IndexFunc(s, notSemverRune); p >= 0 { - s, prerelease = s[:p], s[p:] - } - if strings.HasSuffix(s, ".") { - s += "0" - } else if strings.Count(s, ".") < 2 { - s += ".0" - } - if prerelease != "" { - s += "-" + prerelease - } - return s - } - return "" -} - -func notSemverRune(r rune) bool { - return !strings.ContainsRune("0123456789.", r) -} diff --git a/vendor/cloud.google.com/go/migration.md b/vendor/cloud.google.com/go/migration.md deleted file mode 100644 index 224dcfa13..000000000 --- a/vendor/cloud.google.com/go/migration.md +++ /dev/null @@ -1,50 +0,0 @@ -# go-genproto to google-cloud-go message type migration - -The message types for all of our client libraries are being migrated from the -`google.golang.org/genproto` [module](https://pkg.go.dev/google.golang.org/genproto) -to their respective product specific module in this repository. For example -this asset request type that was once found in [genproto](https://pkg.go.dev/google.golang.org/genproto@v0.0.0-20220908141613-51c1cc9bc6d0/googleapis/cloud/asset/v1p5beta1#ListAssetsRequest) -can now be found in directly in the [asset module](https://pkg.go.dev/cloud.google.com/go/asset/apiv1p5beta1/assetpb#ListAssetsRequest). - -Although the type definitions have moved, aliases have been left in the old -genproto packages to ensure a smooth non-breaking transition. - -## How do I migrate to the new packages? - -The easiest option is to run a migration tool at the root of our project. It is -like `go fix`, but specifically for this migration. Before running the tool it -is best to make sure any modules that have the prefix of `cloud.google.com/go` -are up to date. To run the tool, do the following: - -```bash -go run cloud.google.com/go/internal/aliasfix/cmd/aliasfix@latest . -go mod tidy -``` - -The tool should only change up to one line in the import statement per file. -This can also be done by hand if you prefer. - -## Do I have to migrate? - -Yes if you wish to keep using the newest versions of our client libraries with -the newest features -- You should migrate by the start of 2023. Until then we -will keep updating the aliases in go-genproto weekly. If you have an existing -workload that uses these client libraries and does not need to update its -dependencies there is no action to take. All existing written code will continue -to work. - -## Why are these types being moved - -1. This change will help simplify dependency trees over time. -2. The types will now be in product specific modules that are versioned - independently with semver. This is especially a benefit for users that rely - on multiple clients in a single application. Because message types are no - longer mono-packaged users are less likely to run into intermediate - dependency conflicts when updating dependencies. -3. Having all these types in one repository will help us ensure that unintended - changes are caught before they would be released. - -## Have questions? - -Please reach out to us on our [issue tracker](https://github.com/googleapis/google-cloud-go/issues/new?assignees=&labels=genproto-migration&template=migration-issue.md&title=package%3A+migration+help) -if you have any questions or concerns. diff --git a/vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json b/vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json deleted file mode 100644 index 172016990..000000000 --- a/vendor/cloud.google.com/go/release-please-config-yoshi-submodules.json +++ /dev/null @@ -1,341 +0,0 @@ -{ - "release-type": "go-yoshi", - "include-component-in-tag": true, - "tag-separator": "/", - "packages": { - "accessapproval": { - "component": "accessapproval" - }, - "accesscontextmanager": { - "component": "accesscontextmanager" - }, - "aiplatform": { - "component": "aiplatform" - }, - "analytics": { - "component": "analytics" - }, - "apigateway": { - "component": "apigateway" - }, - "apigeeconnect": { - "component": "apigeeconnect" - }, - "apigeeregistry": { - "component": "apigeeregistry" - }, - "apikeys": { - "component": "apikeys" - }, - "appengine": { - "component": "appengine" - }, - "area120": { - "component": "area120" - }, - "artifactregistry": { - "component": "artifactregistry" - }, - "asset": { - "component": "asset" - }, - "assuredworkloads": { - "component": "assuredworkloads" - }, - "automl": { - "component": "automl" - }, - "baremetalsolution": { - "component": "baremetalsolution" - }, - "batch": { - "component": "batch" - }, - "beyondcorp": { - "component": "beyondcorp" - }, - "billing": { - "component": "billing" - }, - "binaryauthorization": { - "component": "binaryauthorization" - }, - "certificatemanager": { - "component": "certificatemanager" - }, - "channel": { - "component": "channel" - }, - "cloudbuild": { - "component": "cloudbuild" - }, - "clouddms": { - "component": "clouddms" - }, - "cloudtasks": { - "component": "cloudtasks" - }, - "compute": { - "component": "compute" - }, - "compute/metadata": { - "component": "compute/metadata" - }, - "contactcenterinsights": { - "component": "contactcenterinsights" - }, - "container": { - "component": "container" - }, - "containeranalysis": { - "component": "containeranalysis" - }, - "datacatalog": { - "component": "datacatalog" - }, - "dataflow": { - "component": "dataflow" - }, - "dataform": { - "component": "dataform" - }, - "datafusion": { - "component": "datafusion" - }, - "datalabeling": { - "component": "datalabeling" - }, - "dataplex": { - "component": "dataplex" - }, - "dataproc": { - "component": "dataproc" - }, - "dataqna": { - "component": "dataqna" - }, - "datastream": { - "component": "datastream" - }, - "deploy": { - "component": "deploy" - }, - "dialogflow": { - "component": "dialogflow" - }, - "dlp": { - "component": "dlp" - }, - "documentai": { - "component": "documentai" - }, - "domains": { - "component": "domains" - }, - "edgecontainer": { - "component": "edgecontainer" - }, - "essentialcontacts": { - "component": "essentialcontacts" - }, - "eventarc": { - "component": "eventarc" - }, - "filestore": { - "component": "filestore" - }, - "functions": { - "component": "functions" - }, - "gaming": { - "component": "gaming" - }, - "gkebackup": { - "component": "gkebackup" - }, - "gkeconnect": { - "component": "gkeconnect" - }, - "gkehub": { - "component": "gkehub" - }, - "gkemulticloud": { - "component": "gkemulticloud" - }, - "grafeas": { - "component": "grafeas" - }, - "gsuiteaddons": { - "component": "gsuiteaddons" - }, - "iam": { - "component": "iam" - }, - "iap": { - "component": "iap" - }, - "ids": { - "component": "ids" - }, - "iot": { - "component": "iot" - }, - "kms": { - "component": "kms" - }, - "language": { - "component": "language" - }, - "lifesciences": { - "component": "lifesciences" - }, - "managedidentities": { - "component": "managedidentities" - }, - "mediatranslation": { - "component": "mediatranslation" - }, - "memcache": { - "component": "memcache" - }, - "metastore": { - "component": "metastore" - }, - "monitoring": { - "component": "monitoring" - }, - "networkconnectivity": { - "component": "networkconnectivity" - }, - "networkmanagement": { - "component": "networkmanagement" - }, - "networksecurity": { - "component": "networksecurity" - }, - "notebooks": { - "component": "notebooks" - }, - "optimization": { - "component": "optimization" - }, - "orchestration": { - "component": "orchestration" - }, - "orgpolicy": { - "component": "orgpolicy" - }, - "osconfig": { - "component": "osconfig" - }, - "oslogin": { - "component": "oslogin" - }, - "phishingprotection": { - "component": "phishingprotection" - }, - "policytroubleshooter": { - "component": "policytroubleshooter" - }, - "privatecatalog": { - "component": "privatecatalog" - }, - "recaptchaenterprise/v2": { - "component": "recaptchaenterprise" - }, - "recommendationengine": { - "component": "recommendationengine" - }, - "recommender": { - "component": "recommender" - }, - "redis": { - "component": "redis" - }, - "resourcemanager": { - "component": "resourcemanager" - }, - "resourcesettings": { - "component": "resourcesettings" - }, - "retail": { - "component": "retail" - }, - "run": { - "component": "run" - }, - "scheduler": { - "component": "scheduler" - }, - "secretmanager": { - "component": "secretmanager" - }, - "security": { - "component": "security" - }, - "securitycenter": { - "component": "securitycenter" - }, - "servicecontrol": { - "component": "servicecontrol" - }, - "servicedirectory": { - "component": "servicedirectory" - }, - "servicemanagement": { - "component": "servicemanagement" - }, - "serviceusage": { - "component": "serviceusage" - }, - "shell": { - "component": "shell" - }, - "speech": { - "component": "speech" - }, - "storagetransfer": { - "component": "storagetransfer" - }, - "talent": { - "component": "talent" - }, - "texttospeech": { - "component": "texttospeech" - }, - "tpu": { - "component": "tpu" - }, - "trace": { - "component": "trace" - }, - "translate": { - "component": "translate" - }, - "video": { - "component": "video" - }, - "videointelligence": { - "component": "videointelligence" - }, - "vision/v2": { - "component": "vision" - }, - "vmmigration": { - "component": "vmmigration" - }, - "vpcaccess": { - "component": "vpcaccess" - }, - "webrisk": { - "component": "webrisk" - }, - "websecurityscanner": { - "component": "websecurityscanner" - }, - "workflows": { - "component": "workflows" - } - }, - "plugins": ["sentence-case"] -} diff --git a/vendor/cloud.google.com/go/release-please-config.json b/vendor/cloud.google.com/go/release-please-config.json deleted file mode 100644 index 1400245b8..000000000 --- a/vendor/cloud.google.com/go/release-please-config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "release-type": "go-yoshi", - "separate-pull-requests": true, - "include-component-in-tag": false, - "packages": { - ".": { - "component": "main" - } - }, - "plugins": ["sentence-case"] -} diff --git a/vendor/cloud.google.com/go/testing.md b/vendor/cloud.google.com/go/testing.md deleted file mode 100644 index bcca0604d..000000000 --- a/vendor/cloud.google.com/go/testing.md +++ /dev/null @@ -1,236 +0,0 @@ -# Testing Code that depends on Go Client Libraries - -The Go client libraries generated as a part of `cloud.google.com/go` all take -the approach of returning concrete types instead of interfaces. That way, new -fields and methods can be added to the libraries without breaking users. This -document will go over some patterns that can be used to test code that depends -on the Go client libraries. - -## Testing gRPC services using fakes - -*Note*: You can see the full -[example code using a fake here](https://github.com/googleapis/google-cloud-go/tree/main/internal/examples/fake). - -The clients found in `cloud.google.com/go` are gRPC based, with a couple of -notable exceptions being the [`storage`](https://pkg.go.dev/cloud.google.com/go/storage) -and [`bigquery`](https://pkg.go.dev/cloud.google.com/go/bigquery) clients. -Interactions with gRPC services can be faked by serving up your own in-memory -server within your test. One benefit of using this approach is that you don’t -need to define an interface in your runtime code; you can keep using -concrete struct types. You instead define a fake server in your test code. For -example, take a look at the following function: - -```go -import ( - "context" - "fmt" - "log" - "os" - - translate "cloud.google.com/go/translate/apiv3" - "github.com/googleapis/gax-go/v2" - translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" -) - -func TranslateTextWithConcreteClient(client *translate.TranslationClient, text string, targetLang string) (string, error) { - ctx := context.Background() - log.Printf("Translating %q to %q", text, targetLang) - req := &translatepb.TranslateTextRequest{ - Parent: fmt.Sprintf("projects/%s/locations/global", os.Getenv("GOOGLE_CLOUD_PROJECT")), - TargetLanguageCode: "en-US", - Contents: []string{text}, - } - resp, err := client.TranslateText(ctx, req) - if err != nil { - return "", fmt.Errorf("unable to translate text: %v", err) - } - translations := resp.GetTranslations() - if len(translations) != 1 { - return "", fmt.Errorf("expected only one result, got %d", len(translations)) - } - return translations[0].TranslatedText, nil -} -``` - -Here is an example of what a fake server implementation would look like for -faking the interactions above: - -```go -import ( - "context" - - translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" -) - -type fakeTranslationServer struct { - translatepb.UnimplementedTranslationServiceServer -} - -func (f *fakeTranslationServer) TranslateText(ctx context.Context, req *translatepb.TranslateTextRequest) (*translatepb.TranslateTextResponse, error) { - resp := &translatepb.TranslateTextResponse{ - Translations: []*translatepb.Translation{ - &translatepb.Translation{ - TranslatedText: "Hello World", - }, - }, - } - return resp, nil -} -``` - -All of the generated protobuf code found in [google.golang.org/genproto](https://pkg.go.dev/google.golang.org/genproto) -contains a similar `package.UnimplmentedFooServer` type that is useful for -creating fakes. By embedding the unimplemented server in the -`fakeTranslationServer`, the fake will “inherit” all of the RPCs the server -exposes. Then, by providing our own `fakeTranslationServer.TranslateText` -method you can “override” the default unimplemented behavior of the one RPC that -you would like to be faked. - -The test itself does require a little bit of setup: start up a `net.Listener`, -register the server, and tell the client library to call the server: - -```go -import ( - "context" - "net" - "testing" - - translate "cloud.google.com/go/translate/apiv3" - "google.golang.org/api/option" - translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" - "google.golang.org/grpc" -) - -func TestTranslateTextWithConcreteClient(t *testing.T) { - ctx := context.Background() - - // Setup the fake server. - fakeTranslationServer := &fakeTranslationServer{} - l, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatal(err) - } - gsrv := grpc.NewServer() - translatepb.RegisterTranslationServiceServer(gsrv, fakeTranslationServer) - fakeServerAddr := l.Addr().String() - go func() { - if err := gsrv.Serve(l); err != nil { - panic(err) - } - }() - - // Create a client. - client, err := translate.NewTranslationClient(ctx, - option.WithEndpoint(fakeServerAddr), - option.WithoutAuthentication(), - option.WithGRPCDialOption(grpc.WithInsecure()), - ) - if err != nil { - t.Fatal(err) - } - - // Run the test. - text, err := TranslateTextWithConcreteClient(client, "Hola Mundo", "en-US") - if err != nil { - t.Fatal(err) - } - if text != "Hello World" { - t.Fatalf("got %q, want Hello World", text) - } -} -``` - -## Testing using mocks - -*Note*: You can see the full -[example code using a mock here](https://github.com/googleapis/google-cloud-go/tree/main/internal/examples/mock). - -When mocking code you need to work with interfaces. Let’s create an interface -for the `cloud.google.com/go/translate/apiv3` client used in the -`TranslateTextWithConcreteClient` function mentioned in the previous section. -The `translate.Client` has over a dozen methods but this code only uses one of -them. Here is an interface that satisfies the interactions of the -`translate.Client` in this function. - -```go -type TranslationClient interface { - TranslateText(ctx context.Context, req *translatepb.TranslateTextRequest, opts ...gax.CallOption) (*translatepb.TranslateTextResponse, error) -} -``` - -Now that we have an interface that satisfies the method being used we can -rewrite the function signature to take the interface instead of the concrete -type. - -```go -func TranslateTextWithInterfaceClient(client TranslationClient, text string, targetLang string) (string, error) { -// ... -} -``` - -This allows a real `translate.Client` to be passed to the method in production -and for a mock implementation to be passed in during testing. This pattern can -be applied to any Go code, not just `cloud.google.com/go`. This is because -interfaces in Go are implicitly satisfied. Structs in the client libraries can -implicitly implement interfaces defined in your codebase. Let’s take a look at -what it might look like to define a lightweight mock for the `TranslationClient` -interface. - -```go -import ( - "context" - "testing" - - "github.com/googleapis/gax-go/v2" - translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3" -) - -type mockClient struct{} - -func (*mockClient) TranslateText(_ context.Context, req *translatepb.TranslateTextRequest, opts ...gax.CallOption) (*translatepb.TranslateTextResponse, error) { - resp := &translatepb.TranslateTextResponse{ - Translations: []*translatepb.Translation{ - &translatepb.Translation{ - TranslatedText: "Hello World", - }, - }, - } - return resp, nil -} - -func TestTranslateTextWithAbstractClient(t *testing.T) { - client := &mockClient{} - text, err := TranslateTextWithInterfaceClient(client, "Hola Mundo", "en-US") - if err != nil { - t.Fatal(err) - } - if text != "Hello World" { - t.Fatalf("got %q, want Hello World", text) - } -} -``` - -If you prefer to not write your own mocks there are mocking frameworks such as -[golang/mock](https://github.com/golang/mock) which can generate mocks for you -from an interface. As a word of caution though, try to not -[overuse mocks](https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html). - -## Testing using emulators - -Some of the client libraries provided in `cloud.google.com/go` support running -against a service emulator. The concept is similar to that of using fakes, -mentioned above, but the server is managed for you. You just need to start it up -and instruct the client library to talk to the emulator by setting a service -specific emulator environment variable. Current services/environment-variables -are: - -- bigtable: `BIGTABLE_EMULATOR_HOST` -- datastore: `DATASTORE_EMULATOR_HOST` -- firestore: `FIRESTORE_EMULATOR_HOST` -- pubsub: `PUBSUB_EMULATOR_HOST` -- spanner: `SPANNER_EMULATOR_HOST` -- storage: `STORAGE_EMULATOR_HOST` - - Although the storage client supports an emulator environment variable there is no official emulator provided by gcloud. - -For more information on emulators please refer to the -[gcloud documentation](https://cloud.google.com/sdk/gcloud/reference/beta/emulators). diff --git a/vendor/github.com/golang-jwt/jwt/v5/.gitignore b/vendor/github.com/golang-jwt/jwt/v5/.gitignore deleted file mode 100644 index 09573e016..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.DS_Store -bin -.idea/ - diff --git a/vendor/github.com/golang-jwt/jwt/v5/LICENSE b/vendor/github.com/golang-jwt/jwt/v5/LICENSE deleted file mode 100644 index 35dbc2520..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -Copyright (c) 2012 Dave Grijalva -Copyright (c) 2021 golang-jwt maintainers - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md b/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md deleted file mode 100644 index ff9c57e1d..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/MIGRATION_GUIDE.md +++ /dev/null @@ -1,195 +0,0 @@ -# Migration Guide (v5.0.0) - -Version `v5` contains a major rework of core functionalities in the `jwt-go` -library. This includes support for several validation options as well as a -re-design of the `Claims` interface. Lastly, we reworked how errors work under -the hood, which should provide a better overall developer experience. - -Starting from [v5.0.0](https://github.com/golang-jwt/jwt/releases/tag/v5.0.0), -the import path will be: - - "github.com/golang-jwt/jwt/v5" - -For most users, changing the import path *should* suffice. However, since we -intentionally changed and cleaned some of the public API, existing programs -might need to be updated. The following sections describe significant changes -and corresponding updates for existing programs. - -## Parsing and Validation Options - -Under the hood, a new `Validator` struct takes care of validating the claims. A -long awaited feature has been the option to fine-tune the validation of tokens. -This is now possible with several `ParserOption` functions that can be appended -to most `Parse` functions, such as `ParseWithClaims`. The most important options -and changes are: - * Added `WithLeeway` to support specifying the leeway that is allowed when - validating time-based claims, such as `exp` or `nbf`. - * Changed default behavior to not check the `iat` claim. Usage of this claim - is OPTIONAL according to the JWT RFC. The claim itself is also purely - informational according to the RFC, so a strict validation failure is not - recommended. If you want to check for sensible values in these claims, - please use the `WithIssuedAt` parser option. - * Added `WithAudience`, `WithSubject` and `WithIssuer` to support checking for - expected `aud`, `sub` and `iss`. - * Added `WithStrictDecoding` and `WithPaddingAllowed` options to allow - previously global settings to enable base64 strict encoding and the parsing - of base64 strings with padding. The latter is strictly speaking against the - standard, but unfortunately some of the major identity providers issue some - of these incorrect tokens. Both options are disabled by default. - -## Changes to the `Claims` interface - -### Complete Restructuring - -Previously, the claims interface was satisfied with an implementation of a -`Valid() error` function. This had several issues: - * The different claim types (struct claims, map claims, etc.) then contained - similar (but not 100 % identical) code of how this validation was done. This - lead to a lot of (almost) duplicate code and was hard to maintain - * It was not really semantically close to what a "claim" (or a set of claims) - really is; which is a list of defined key/value pairs with a certain - semantic meaning. - -Since all the validation functionality is now extracted into the validator, all -`VerifyXXX` and `Valid` functions have been removed from the `Claims` interface. -Instead, the interface now represents a list of getters to retrieve values with -a specific meaning. This allows us to completely decouple the validation logic -with the underlying storage representation of the claim, which could be a -struct, a map or even something stored in a database. - -```go -type Claims interface { - GetExpirationTime() (*NumericDate, error) - GetIssuedAt() (*NumericDate, error) - GetNotBefore() (*NumericDate, error) - GetIssuer() (string, error) - GetSubject() (string, error) - GetAudience() (ClaimStrings, error) -} -``` - -Users that previously directly called the `Valid` function on their claims, -e.g., to perform validation independently of parsing/verifying a token, can now -use the `jwt.NewValidator` function to create a `Validator` independently of the -`Parser`. - -```go -var v = jwt.NewValidator(jwt.WithLeeway(5*time.Second)) -v.Validate(myClaims) -``` - -### Supported Claim Types and Removal of `StandardClaims` - -The two standard claim types supported by this library, `MapClaims` and -`RegisteredClaims` both implement the necessary functions of this interface. The -old `StandardClaims` struct, which has already been deprecated in `v4` is now -removed. - -Users using custom claims, in most cases, will not experience any changes in the -behavior as long as they embedded `RegisteredClaims`. If they created a new -claim type from scratch, they now need to implemented the proper getter -functions. - -### Migrating Application Specific Logic of the old `Valid` - -Previously, users could override the `Valid` method in a custom claim, for -example to extend the validation with application-specific claims. However, this -was always very dangerous, since once could easily disable the standard -validation and signature checking. - -In order to avoid that, while still supporting the use-case, a new -`ClaimsValidator` interface has been introduced. This interface consists of the -`Validate() error` function. If the validator sees, that a `Claims` struct -implements this interface, the errors returned to the `Validate` function will -be *appended* to the regular standard validation. It is not possible to disable -the standard validation anymore (even only by accident). - -Usage examples can be found in [example_test.go](./example_test.go), to build -claims structs like the following. - -```go -// MyCustomClaims includes all registered claims, plus Foo. -type MyCustomClaims struct { - Foo string `json:"foo"` - jwt.RegisteredClaims -} - -// Validate can be used to execute additional application-specific claims -// validation. -func (m MyCustomClaims) Validate() error { - if m.Foo != "bar" { - return errors.New("must be foobar") - } - - return nil -} -``` - -## Changes to the `Token` and `Parser` struct - -The previously global functions `DecodeSegment` and `EncodeSegment` were moved -to the `Parser` and `Token` struct respectively. This will allow us in the -future to configure the behavior of these two based on options supplied on the -parser or the token (creation). This also removes two previously global -variables and moves them to parser options `WithStrictDecoding` and -`WithPaddingAllowed`. - -In order to do that, we had to adjust the way signing methods work. Previously -they were given a base64 encoded signature in `Verify` and were expected to -return a base64 encoded version of the signature in `Sign`, both as a `string`. -However, this made it necessary to have `DecodeSegment` and `EncodeSegment` -global and was a less than perfect design because we were repeating -encoding/decoding steps for all signing methods. Now, `Sign` and `Verify` -operate on a decoded signature as a `[]byte`, which feels more natural for a -cryptographic operation anyway. Lastly, `Parse` and `SignedString` take care of -the final encoding/decoding part. - -In addition to that, we also changed the `Signature` field on `Token` from a -`string` to `[]byte` and this is also now populated with the decoded form. This -is also more consistent, because the other parts of the JWT, mainly `Header` and -`Claims` were already stored in decoded form in `Token`. Only the signature was -stored in base64 encoded form, which was redundant with the information in the -`Raw` field, which contains the complete token as base64. - -```go -type Token struct { - Raw string // Raw contains the raw token - Method SigningMethod // Method is the signing method used or to be used - Header map[string]interface{} // Header is the first segment of the token in decoded form - Claims Claims // Claims is the second segment of the token in decoded form - Signature []byte // Signature is the third segment of the token in decoded form - Valid bool // Valid specifies if the token is valid -} -``` - -Most (if not all) of these changes should not impact the normal usage of this -library. Only users directly accessing the `Signature` field as well as -developers of custom signing methods should be affected. - -# Migration Guide (v4.0.0) - -Starting from [v4.0.0](https://github.com/golang-jwt/jwt/releases/tag/v4.0.0), -the import path will be: - - "github.com/golang-jwt/jwt/v4" - -The `/v4` version will be backwards compatible with existing `v3.x.y` tags in -this repo, as well as `github.com/dgrijalva/jwt-go`. For most users this should -be a drop-in replacement, if you're having troubles migrating, please open an -issue. - -You can replace all occurrences of `github.com/dgrijalva/jwt-go` or -`github.com/golang-jwt/jwt` with `github.com/golang-jwt/jwt/v4`, either manually -or by using tools such as `sed` or `gofmt`. - -And then you'd typically run: - -``` -go get github.com/golang-jwt/jwt/v4 -go mod tidy -``` - -# Older releases (before v3.2.0) - -The original migration guide for older releases can be found at -https://github.com/dgrijalva/jwt-go/blob/master/MIGRATION_GUIDE.md. diff --git a/vendor/github.com/golang-jwt/jwt/v5/README.md b/vendor/github.com/golang-jwt/jwt/v5/README.md deleted file mode 100644 index 964598a31..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/README.md +++ /dev/null @@ -1,167 +0,0 @@ -# jwt-go - -[![build](https://github.com/golang-jwt/jwt/actions/workflows/build.yml/badge.svg)](https://github.com/golang-jwt/jwt/actions/workflows/build.yml) -[![Go -Reference](https://pkg.go.dev/badge/github.com/golang-jwt/jwt/v5.svg)](https://pkg.go.dev/github.com/golang-jwt/jwt/v5) -[![Coverage Status](https://coveralls.io/repos/github/golang-jwt/jwt/badge.svg?branch=main)](https://coveralls.io/github/golang-jwt/jwt?branch=main) - -A [go](http://www.golang.org) (or 'golang' for search engine friendliness) -implementation of [JSON Web -Tokens](https://datatracker.ietf.org/doc/html/rfc7519). - -Starting with [v4.0.0](https://github.com/golang-jwt/jwt/releases/tag/v4.0.0) -this project adds Go module support, but maintains backwards compatibility with -older `v3.x.y` tags and upstream `github.com/dgrijalva/jwt-go`. See the -[`MIGRATION_GUIDE.md`](./MIGRATION_GUIDE.md) for more information. Version -v5.0.0 introduces major improvements to the validation of tokens, but is not -entirely backwards compatible. - -> After the original author of the library suggested migrating the maintenance -> of `jwt-go`, a dedicated team of open source maintainers decided to clone the -> existing library into this repository. See -> [dgrijalva/jwt-go#462](https://github.com/dgrijalva/jwt-go/issues/462) for a -> detailed discussion on this topic. - - -**SECURITY NOTICE:** Some older versions of Go have a security issue in the -crypto/elliptic. Recommendation is to upgrade to at least 1.15 See issue -[dgrijalva/jwt-go#216](https://github.com/dgrijalva/jwt-go/issues/216) for more -detail. - -**SECURITY NOTICE:** It's important that you [validate the `alg` presented is -what you -expect](https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/). -This library attempts to make it easy to do the right thing by requiring key -types match the expected alg, but you should take the extra step to verify it in -your usage. See the examples provided. - -### Supported Go versions - -Our support of Go versions is aligned with Go's [version release -policy](https://golang.org/doc/devel/release#policy). So we will support a major -version of Go until there are two newer major releases. We no longer support -building jwt-go with unsupported Go versions, as these contain security -vulnerabilities which will not be fixed. - -## What the heck is a JWT? - -JWT.io has [a great introduction](https://jwt.io/introduction) to JSON Web -Tokens. - -In short, it's a signed JSON object that does something useful (for example, -authentication). It's commonly used for `Bearer` tokens in Oauth 2. A token is -made of three parts, separated by `.`'s. The first two parts are JSON objects, -that have been [base64url](https://datatracker.ietf.org/doc/html/rfc4648) -encoded. The last part is the signature, encoded the same way. - -The first part is called the header. It contains the necessary information for -verifying the last part, the signature. For example, which encryption method -was used for signing and what key was used. - -The part in the middle is the interesting bit. It's called the Claims and -contains the actual stuff you care about. Refer to [RFC -7519](https://datatracker.ietf.org/doc/html/rfc7519) for information about -reserved keys and the proper way to add your own. - -## What's in the box? - -This library supports the parsing and verification as well as the generation and -signing of JWTs. Current supported signing algorithms are HMAC SHA, RSA, -RSA-PSS, and ECDSA, though hooks are present for adding your own. - -## Installation Guidelines - -1. To install the jwt package, you first need to have - [Go](https://go.dev/doc/install) installed, then you can use the command - below to add `jwt-go` as a dependency in your Go program. - -```sh -go get -u github.com/golang-jwt/jwt/v5 -``` - -2. Import it in your code: - -```go -import "github.com/golang-jwt/jwt/v5" -``` - -## Usage - -A detailed usage guide, including how to sign and verify tokens can be found on -our [documentation website](https://golang-jwt.github.io/jwt/usage/create/). - -## Examples - -See [the project documentation](https://pkg.go.dev/github.com/golang-jwt/jwt/v5) -for examples of usage: - -* [Simple example of parsing and validating a - token](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#example-Parse-Hmac) -* [Simple example of building and signing a - token](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#example-New-Hmac) -* [Directory of - Examples](https://pkg.go.dev/github.com/golang-jwt/jwt/v5#pkg-examples) - -## Compliance - -This library was last reviewed to comply with [RFC -7519](https://datatracker.ietf.org/doc/html/rfc7519) dated May 2015 with a few -notable differences: - -* In order to protect against accidental use of [Unsecured - JWTs](https://datatracker.ietf.org/doc/html/rfc7519#section-6), tokens using - `alg=none` will only be accepted if the constant - `jwt.UnsafeAllowNoneSignatureType` is provided as the key. - -## Project Status & Versioning - -This library is considered production ready. Feedback and feature requests are -appreciated. The API should be considered stable. There should be very few -backwards-incompatible changes outside of major version updates (and only with -good reason). - -This project uses [Semantic Versioning 2.0.0](http://semver.org). Accepted pull -requests will land on `main`. Periodically, versions will be tagged from -`main`. You can find all the releases on [the project releases -page](https://github.com/golang-jwt/jwt/releases). - -**BREAKING CHANGES:*** A full list of breaking changes is available in -`VERSION_HISTORY.md`. See `MIGRATION_GUIDE.md` for more information on updating -your code. - -## Extensions - -This library publishes all the necessary components for adding your own signing -methods or key functions. Simply implement the `SigningMethod` interface and -register a factory method using `RegisterSigningMethod` or provide a -`jwt.Keyfunc`. - -A common use case would be integrating with different 3rd party signature -providers, like key management services from various cloud providers or Hardware -Security Modules (HSMs) or to implement additional standards. - -| Extension | Purpose | Repo | -| --------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------ | -| GCP | Integrates with multiple Google Cloud Platform signing tools (AppEngine, IAM API, Cloud KMS) | https://github.com/someone1/gcp-jwt-go | -| AWS | Integrates with AWS Key Management Service, KMS | https://github.com/matelang/jwt-go-aws-kms | -| JWKS | Provides support for JWKS ([RFC 7517](https://datatracker.ietf.org/doc/html/rfc7517)) as a `jwt.Keyfunc` | https://github.com/MicahParks/keyfunc | - -*Disclaimer*: Unless otherwise specified, these integrations are maintained by -third parties and should not be considered as a primary offer by any of the -mentioned cloud providers - -## More - -Go package documentation can be found [on -pkg.go.dev](https://pkg.go.dev/github.com/golang-jwt/jwt/v5). Additional -documentation can be found on [our project -page](https://golang-jwt.github.io/jwt/). - -The command line utility included in this project (cmd/jwt) provides a -straightforward example of token creation and parsing as well as a useful tool -for debugging your own integration. You'll also find several implementation -examples in the documentation. - -[golang-jwt](https://github.com/orgs/golang-jwt) incorporates a modified version -of the JWT logo, which is distributed under the terms of the [MIT -License](https://github.com/jsonwebtoken/jsonwebtoken.github.io/blob/master/LICENSE.txt). diff --git a/vendor/github.com/golang-jwt/jwt/v5/SECURITY.md b/vendor/github.com/golang-jwt/jwt/v5/SECURITY.md deleted file mode 100644 index b08402c34..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/SECURITY.md +++ /dev/null @@ -1,19 +0,0 @@ -# Security Policy - -## Supported Versions - -As of February 2022 (and until this document is updated), the latest version `v4` is supported. - -## Reporting a Vulnerability - -If you think you found a vulnerability, and even if you are not sure, please report it to jwt-go-security@googlegroups.com or one of the other [golang-jwt maintainers](https://github.com/orgs/golang-jwt/people). Please try be explicit, describe steps to reproduce the security issue with code example(s). - -You will receive a response within a timely manner. If the issue is confirmed, we will do our best to release a patch as soon as possible given the complexity of the problem. - -## Public Discussions - -Please avoid publicly discussing a potential security vulnerability. - -Let's take this offline and find a solution first, this limits the potential impact as much as possible. - -We appreciate your help! diff --git a/vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md b/vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md deleted file mode 100644 index b5039e49c..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/VERSION_HISTORY.md +++ /dev/null @@ -1,137 +0,0 @@ -# `jwt-go` Version History - -The following version history is kept for historic purposes. To retrieve the current changes of each version, please refer to the change-log of the specific release versions on https://github.com/golang-jwt/jwt/releases. - -## 4.0.0 - -* Introduces support for Go modules. The `v4` version will be backwards compatible with `v3.x.y`. - -## 3.2.2 - -* Starting from this release, we are adopting the policy to support the most 2 recent versions of Go currently available. By the time of this release, this is Go 1.15 and 1.16 ([#28](https://github.com/golang-jwt/jwt/pull/28)). -* Fixed a potential issue that could occur when the verification of `exp`, `iat` or `nbf` was not required and contained invalid contents, i.e. non-numeric/date. Thanks for @thaJeztah for making us aware of that and @giorgos-f3 for originally reporting it to the formtech fork ([#40](https://github.com/golang-jwt/jwt/pull/40)). -* Added support for EdDSA / ED25519 ([#36](https://github.com/golang-jwt/jwt/pull/36)). -* Optimized allocations ([#33](https://github.com/golang-jwt/jwt/pull/33)). - -## 3.2.1 - -* **Import Path Change**: See MIGRATION_GUIDE.md for tips on updating your code - * Changed the import path from `github.com/dgrijalva/jwt-go` to `github.com/golang-jwt/jwt` -* Fixed type confusing issue between `string` and `[]string` in `VerifyAudience` ([#12](https://github.com/golang-jwt/jwt/pull/12)). This fixes CVE-2020-26160 - -#### 3.2.0 - -* Added method `ParseUnverified` to allow users to split up the tasks of parsing and validation -* HMAC signing method returns `ErrInvalidKeyType` instead of `ErrInvalidKey` where appropriate -* Added options to `request.ParseFromRequest`, which allows for an arbitrary list of modifiers to parsing behavior. Initial set include `WithClaims` and `WithParser`. Existing usage of this function will continue to work as before. -* Deprecated `ParseFromRequestWithClaims` to simplify API in the future. - -#### 3.1.0 - -* Improvements to `jwt` command line tool -* Added `SkipClaimsValidation` option to `Parser` -* Documentation updates - -#### 3.0.0 - -* **Compatibility Breaking Changes**: See MIGRATION_GUIDE.md for tips on updating your code - * Dropped support for `[]byte` keys when using RSA signing methods. This convenience feature could contribute to security vulnerabilities involving mismatched key types with signing methods. - * `ParseFromRequest` has been moved to `request` subpackage and usage has changed - * The `Claims` property on `Token` is now type `Claims` instead of `map[string]interface{}`. The default value is type `MapClaims`, which is an alias to `map[string]interface{}`. This makes it possible to use a custom type when decoding claims. -* Other Additions and Changes - * Added `Claims` interface type to allow users to decode the claims into a custom type - * Added `ParseWithClaims`, which takes a third argument of type `Claims`. Use this function instead of `Parse` if you have a custom type you'd like to decode into. - * Dramatically improved the functionality and flexibility of `ParseFromRequest`, which is now in the `request` subpackage - * Added `ParseFromRequestWithClaims` which is the `FromRequest` equivalent of `ParseWithClaims` - * Added new interface type `Extractor`, which is used for extracting JWT strings from http requests. Used with `ParseFromRequest` and `ParseFromRequestWithClaims`. - * Added several new, more specific, validation errors to error type bitmask - * Moved examples from README to executable example files - * Signing method registry is now thread safe - * Added new property to `ValidationError`, which contains the raw error returned by calls made by parse/verify (such as those returned by keyfunc or json parser) - -#### 2.7.0 - -This will likely be the last backwards compatible release before 3.0.0, excluding essential bug fixes. - -* Added new option `-show` to the `jwt` command that will just output the decoded token without verifying -* Error text for expired tokens includes how long it's been expired -* Fixed incorrect error returned from `ParseRSAPublicKeyFromPEM` -* Documentation updates - -#### 2.6.0 - -* Exposed inner error within ValidationError -* Fixed validation errors when using UseJSONNumber flag -* Added several unit tests - -#### 2.5.0 - -* Added support for signing method none. You shouldn't use this. The API tries to make this clear. -* Updated/fixed some documentation -* Added more helpful error message when trying to parse tokens that begin with `BEARER ` - -#### 2.4.0 - -* Added new type, Parser, to allow for configuration of various parsing parameters - * You can now specify a list of valid signing methods. Anything outside this set will be rejected. - * You can now opt to use the `json.Number` type instead of `float64` when parsing token JSON -* Added support for [Travis CI](https://travis-ci.org/dgrijalva/jwt-go) -* Fixed some bugs with ECDSA parsing - -#### 2.3.0 - -* Added support for ECDSA signing methods -* Added support for RSA PSS signing methods (requires go v1.4) - -#### 2.2.0 - -* Gracefully handle a `nil` `Keyfunc` being passed to `Parse`. Result will now be the parsed token and an error, instead of a panic. - -#### 2.1.0 - -Backwards compatible API change that was missed in 2.0.0. - -* The `SignedString` method on `Token` now takes `interface{}` instead of `[]byte` - -#### 2.0.0 - -There were two major reasons for breaking backwards compatibility with this update. The first was a refactor required to expand the width of the RSA and HMAC-SHA signing implementations. There will likely be no required code changes to support this change. - -The second update, while unfortunately requiring a small change in integration, is required to open up this library to other signing methods. Not all keys used for all signing methods have a single standard on-disk representation. Requiring `[]byte` as the type for all keys proved too limiting. Additionally, this implementation allows for pre-parsed tokens to be reused, which might matter in an application that parses a high volume of tokens with a small set of keys. Backwards compatibilty has been maintained for passing `[]byte` to the RSA signing methods, but they will also accept `*rsa.PublicKey` and `*rsa.PrivateKey`. - -It is likely the only integration change required here will be to change `func(t *jwt.Token) ([]byte, error)` to `func(t *jwt.Token) (interface{}, error)` when calling `Parse`. - -* **Compatibility Breaking Changes** - * `SigningMethodHS256` is now `*SigningMethodHMAC` instead of `type struct` - * `SigningMethodRS256` is now `*SigningMethodRSA` instead of `type struct` - * `KeyFunc` now returns `interface{}` instead of `[]byte` - * `SigningMethod.Sign` now takes `interface{}` instead of `[]byte` for the key - * `SigningMethod.Verify` now takes `interface{}` instead of `[]byte` for the key -* Renamed type `SigningMethodHS256` to `SigningMethodHMAC`. Specific sizes are now just instances of this type. - * Added public package global `SigningMethodHS256` - * Added public package global `SigningMethodHS384` - * Added public package global `SigningMethodHS512` -* Renamed type `SigningMethodRS256` to `SigningMethodRSA`. Specific sizes are now just instances of this type. - * Added public package global `SigningMethodRS256` - * Added public package global `SigningMethodRS384` - * Added public package global `SigningMethodRS512` -* Moved sample private key for HMAC tests from an inline value to a file on disk. Value is unchanged. -* Refactored the RSA implementation to be easier to read -* Exposed helper methods `ParseRSAPrivateKeyFromPEM` and `ParseRSAPublicKeyFromPEM` - -## 1.0.2 - -* Fixed bug in parsing public keys from certificates -* Added more tests around the parsing of keys for RS256 -* Code refactoring in RS256 implementation. No functional changes - -## 1.0.1 - -* Fixed panic if RS256 signing method was passed an invalid key - -## 1.0.0 - -* First versioned release -* API stabilized -* Supports creating, signing, parsing, and validating JWT tokens -* Supports RS256 and HS256 signing methods diff --git a/vendor/github.com/golang-jwt/jwt/v5/claims.go b/vendor/github.com/golang-jwt/jwt/v5/claims.go deleted file mode 100644 index d50ff3dad..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/claims.go +++ /dev/null @@ -1,16 +0,0 @@ -package jwt - -// Claims represent any form of a JWT Claims Set according to -// https://datatracker.ietf.org/doc/html/rfc7519#section-4. In order to have a -// common basis for validation, it is required that an implementation is able to -// supply at least the claim names provided in -// https://datatracker.ietf.org/doc/html/rfc7519#section-4.1 namely `exp`, -// `iat`, `nbf`, `iss`, `sub` and `aud`. -type Claims interface { - GetExpirationTime() (*NumericDate, error) - GetIssuedAt() (*NumericDate, error) - GetNotBefore() (*NumericDate, error) - GetIssuer() (string, error) - GetSubject() (string, error) - GetAudience() (ClaimStrings, error) -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/doc.go b/vendor/github.com/golang-jwt/jwt/v5/doc.go deleted file mode 100644 index a86dc1a3b..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package jwt is a Go implementation of JSON Web Tokens: http://self-issued.info/docs/draft-jones-json-web-token.html -// -// See README.md for more info. -package jwt diff --git a/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go b/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go deleted file mode 100644 index c929e4a02..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/ecdsa.go +++ /dev/null @@ -1,134 +0,0 @@ -package jwt - -import ( - "crypto" - "crypto/ecdsa" - "crypto/rand" - "errors" - "math/big" -) - -var ( - // Sadly this is missing from crypto/ecdsa compared to crypto/rsa - ErrECDSAVerification = errors.New("crypto/ecdsa: verification error") -) - -// SigningMethodECDSA implements the ECDSA family of signing methods. -// Expects *ecdsa.PrivateKey for signing and *ecdsa.PublicKey for verification -type SigningMethodECDSA struct { - Name string - Hash crypto.Hash - KeySize int - CurveBits int -} - -// Specific instances for EC256 and company -var ( - SigningMethodES256 *SigningMethodECDSA - SigningMethodES384 *SigningMethodECDSA - SigningMethodES512 *SigningMethodECDSA -) - -func init() { - // ES256 - SigningMethodES256 = &SigningMethodECDSA{"ES256", crypto.SHA256, 32, 256} - RegisterSigningMethod(SigningMethodES256.Alg(), func() SigningMethod { - return SigningMethodES256 - }) - - // ES384 - SigningMethodES384 = &SigningMethodECDSA{"ES384", crypto.SHA384, 48, 384} - RegisterSigningMethod(SigningMethodES384.Alg(), func() SigningMethod { - return SigningMethodES384 - }) - - // ES512 - SigningMethodES512 = &SigningMethodECDSA{"ES512", crypto.SHA512, 66, 521} - RegisterSigningMethod(SigningMethodES512.Alg(), func() SigningMethod { - return SigningMethodES512 - }) -} - -func (m *SigningMethodECDSA) Alg() string { - return m.Name -} - -// Verify implements token verification for the SigningMethod. -// For this verify method, key must be an ecdsa.PublicKey struct -func (m *SigningMethodECDSA) Verify(signingString string, sig []byte, key interface{}) error { - // Get the key - var ecdsaKey *ecdsa.PublicKey - switch k := key.(type) { - case *ecdsa.PublicKey: - ecdsaKey = k - default: - return newError("ECDSA verify expects *ecdsa.PublicKey", ErrInvalidKeyType) - } - - if len(sig) != 2*m.KeySize { - return ErrECDSAVerification - } - - r := big.NewInt(0).SetBytes(sig[:m.KeySize]) - s := big.NewInt(0).SetBytes(sig[m.KeySize:]) - - // Create hasher - if !m.Hash.Available() { - return ErrHashUnavailable - } - hasher := m.Hash.New() - hasher.Write([]byte(signingString)) - - // Verify the signature - if verifystatus := ecdsa.Verify(ecdsaKey, hasher.Sum(nil), r, s); verifystatus { - return nil - } - - return ErrECDSAVerification -} - -// Sign implements token signing for the SigningMethod. -// For this signing method, key must be an ecdsa.PrivateKey struct -func (m *SigningMethodECDSA) Sign(signingString string, key interface{}) ([]byte, error) { - // Get the key - var ecdsaKey *ecdsa.PrivateKey - switch k := key.(type) { - case *ecdsa.PrivateKey: - ecdsaKey = k - default: - return nil, newError("ECDSA sign expects *ecdsa.PrivateKey", ErrInvalidKeyType) - } - - // Create the hasher - if !m.Hash.Available() { - return nil, ErrHashUnavailable - } - - hasher := m.Hash.New() - hasher.Write([]byte(signingString)) - - // Sign the string and return r, s - if r, s, err := ecdsa.Sign(rand.Reader, ecdsaKey, hasher.Sum(nil)); err == nil { - curveBits := ecdsaKey.Curve.Params().BitSize - - if m.CurveBits != curveBits { - return nil, ErrInvalidKey - } - - keyBytes := curveBits / 8 - if curveBits%8 > 0 { - keyBytes += 1 - } - - // We serialize the outputs (r and s) into big-endian byte arrays - // padded with zeros on the left to make sure the sizes work out. - // Output must be 2*keyBytes long. - out := make([]byte, 2*keyBytes) - r.FillBytes(out[0:keyBytes]) // r is assigned to the first half of output. - s.FillBytes(out[keyBytes:]) // s is assigned to the second half of output. - - return out, nil - } else { - return nil, err - } -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go b/vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go deleted file mode 100644 index 5700636d3..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/ecdsa_utils.go +++ /dev/null @@ -1,69 +0,0 @@ -package jwt - -import ( - "crypto/ecdsa" - "crypto/x509" - "encoding/pem" - "errors" -) - -var ( - ErrNotECPublicKey = errors.New("key is not a valid ECDSA public key") - ErrNotECPrivateKey = errors.New("key is not a valid ECDSA private key") -) - -// ParseECPrivateKeyFromPEM parses a PEM encoded Elliptic Curve Private Key Structure -func ParseECPrivateKeyFromPEM(key []byte) (*ecdsa.PrivateKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - // Parse the key - var parsedKey interface{} - if parsedKey, err = x509.ParseECPrivateKey(block.Bytes); err != nil { - if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { - return nil, err - } - } - - var pkey *ecdsa.PrivateKey - var ok bool - if pkey, ok = parsedKey.(*ecdsa.PrivateKey); !ok { - return nil, ErrNotECPrivateKey - } - - return pkey, nil -} - -// ParseECPublicKeyFromPEM parses a PEM encoded PKCS1 or PKCS8 public key -func ParseECPublicKeyFromPEM(key []byte) (*ecdsa.PublicKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - // Parse the key - var parsedKey interface{} - if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { - if cert, err := x509.ParseCertificate(block.Bytes); err == nil { - parsedKey = cert.PublicKey - } else { - return nil, err - } - } - - var pkey *ecdsa.PublicKey - var ok bool - if pkey, ok = parsedKey.(*ecdsa.PublicKey); !ok { - return nil, ErrNotECPublicKey - } - - return pkey, nil -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/ed25519.go b/vendor/github.com/golang-jwt/jwt/v5/ed25519.go deleted file mode 100644 index c2138119e..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/ed25519.go +++ /dev/null @@ -1,79 +0,0 @@ -package jwt - -import ( - "crypto" - "crypto/ed25519" - "crypto/rand" - "errors" -) - -var ( - ErrEd25519Verification = errors.New("ed25519: verification error") -) - -// SigningMethodEd25519 implements the EdDSA family. -// Expects ed25519.PrivateKey for signing and ed25519.PublicKey for verification -type SigningMethodEd25519 struct{} - -// Specific instance for EdDSA -var ( - SigningMethodEdDSA *SigningMethodEd25519 -) - -func init() { - SigningMethodEdDSA = &SigningMethodEd25519{} - RegisterSigningMethod(SigningMethodEdDSA.Alg(), func() SigningMethod { - return SigningMethodEdDSA - }) -} - -func (m *SigningMethodEd25519) Alg() string { - return "EdDSA" -} - -// Verify implements token verification for the SigningMethod. -// For this verify method, key must be an ed25519.PublicKey -func (m *SigningMethodEd25519) Verify(signingString string, sig []byte, key interface{}) error { - var ed25519Key ed25519.PublicKey - var ok bool - - if ed25519Key, ok = key.(ed25519.PublicKey); !ok { - return newError("Ed25519 verify expects ed25519.PublicKey", ErrInvalidKeyType) - } - - if len(ed25519Key) != ed25519.PublicKeySize { - return ErrInvalidKey - } - - // Verify the signature - if !ed25519.Verify(ed25519Key, []byte(signingString), sig) { - return ErrEd25519Verification - } - - return nil -} - -// Sign implements token signing for the SigningMethod. -// For this signing method, key must be an ed25519.PrivateKey -func (m *SigningMethodEd25519) Sign(signingString string, key interface{}) ([]byte, error) { - var ed25519Key crypto.Signer - var ok bool - - if ed25519Key, ok = key.(crypto.Signer); !ok { - return nil, newError("Ed25519 sign expects crypto.Signer", ErrInvalidKeyType) - } - - if _, ok := ed25519Key.Public().(ed25519.PublicKey); !ok { - return nil, ErrInvalidKey - } - - // Sign the string and return the result. ed25519 performs a two-pass hash - // as part of its algorithm. Therefore, we need to pass a non-prehashed - // message into the Sign function, as indicated by crypto.Hash(0) - sig, err := ed25519Key.Sign(rand.Reader, []byte(signingString), crypto.Hash(0)) - if err != nil { - return nil, err - } - - return sig, nil -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go b/vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go deleted file mode 100644 index cdb5e68e8..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/ed25519_utils.go +++ /dev/null @@ -1,64 +0,0 @@ -package jwt - -import ( - "crypto" - "crypto/ed25519" - "crypto/x509" - "encoding/pem" - "errors" -) - -var ( - ErrNotEdPrivateKey = errors.New("key is not a valid Ed25519 private key") - ErrNotEdPublicKey = errors.New("key is not a valid Ed25519 public key") -) - -// ParseEdPrivateKeyFromPEM parses a PEM-encoded Edwards curve private key -func ParseEdPrivateKeyFromPEM(key []byte) (crypto.PrivateKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - // Parse the key - var parsedKey interface{} - if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { - return nil, err - } - - var pkey ed25519.PrivateKey - var ok bool - if pkey, ok = parsedKey.(ed25519.PrivateKey); !ok { - return nil, ErrNotEdPrivateKey - } - - return pkey, nil -} - -// ParseEdPublicKeyFromPEM parses a PEM-encoded Edwards curve public key -func ParseEdPublicKeyFromPEM(key []byte) (crypto.PublicKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - // Parse the key - var parsedKey interface{} - if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { - return nil, err - } - - var pkey ed25519.PublicKey - var ok bool - if pkey, ok = parsedKey.(ed25519.PublicKey); !ok { - return nil, ErrNotEdPublicKey - } - - return pkey, nil -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors.go b/vendor/github.com/golang-jwt/jwt/v5/errors.go deleted file mode 100644 index 23bb616dd..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/errors.go +++ /dev/null @@ -1,49 +0,0 @@ -package jwt - -import ( - "errors" - "strings" -) - -var ( - ErrInvalidKey = errors.New("key is invalid") - ErrInvalidKeyType = errors.New("key is of invalid type") - ErrHashUnavailable = errors.New("the requested hash function is unavailable") - ErrTokenMalformed = errors.New("token is malformed") - ErrTokenUnverifiable = errors.New("token is unverifiable") - ErrTokenSignatureInvalid = errors.New("token signature is invalid") - ErrTokenRequiredClaimMissing = errors.New("token is missing required claim") - ErrTokenInvalidAudience = errors.New("token has invalid audience") - ErrTokenExpired = errors.New("token is expired") - ErrTokenUsedBeforeIssued = errors.New("token used before issued") - ErrTokenInvalidIssuer = errors.New("token has invalid issuer") - ErrTokenInvalidSubject = errors.New("token has invalid subject") - ErrTokenNotValidYet = errors.New("token is not valid yet") - ErrTokenInvalidId = errors.New("token has invalid id") - ErrTokenInvalidClaims = errors.New("token has invalid claims") - ErrInvalidType = errors.New("invalid type for claim") -) - -// joinedError is an error type that works similar to what [errors.Join] -// produces, with the exception that it has a nice error string; mainly its -// error messages are concatenated using a comma, rather than a newline. -type joinedError struct { - errs []error -} - -func (je joinedError) Error() string { - msg := []string{} - for _, err := range je.errs { - msg = append(msg, err.Error()) - } - - return strings.Join(msg, ", ") -} - -// joinErrors joins together multiple errors. Useful for scenarios where -// multiple errors next to each other occur, e.g., in claims validation. -func joinErrors(errs ...error) error { - return &joinedError{ - errs: errs, - } -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go b/vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go deleted file mode 100644 index a893d355e..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/errors_go1_20.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build go1.20 -// +build go1.20 - -package jwt - -import ( - "fmt" -) - -// Unwrap implements the multiple error unwrapping for this error type, which is -// possible in Go 1.20. -func (je joinedError) Unwrap() []error { - return je.errs -} - -// newError creates a new error message with a detailed error message. The -// message will be prefixed with the contents of the supplied error type. -// Additionally, more errors, that provide more context can be supplied which -// will be appended to the message. This makes use of Go 1.20's possibility to -// include more than one %w formatting directive in [fmt.Errorf]. -// -// For example, -// -// newError("no keyfunc was provided", ErrTokenUnverifiable) -// -// will produce the error string -// -// "token is unverifiable: no keyfunc was provided" -func newError(message string, err error, more ...error) error { - var format string - var args []any - if message != "" { - format = "%w: %s" - args = []any{err, message} - } else { - format = "%w" - args = []any{err} - } - - for _, e := range more { - format += ": %w" - args = append(args, e) - } - - err = fmt.Errorf(format, args...) - return err -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go b/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go deleted file mode 100644 index 2ad542f00..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/errors_go_other.go +++ /dev/null @@ -1,78 +0,0 @@ -//go:build !go1.20 -// +build !go1.20 - -package jwt - -import ( - "errors" - "fmt" -) - -// Is implements checking for multiple errors using [errors.Is], since multiple -// error unwrapping is not possible in versions less than Go 1.20. -func (je joinedError) Is(err error) bool { - for _, e := range je.errs { - if errors.Is(e, err) { - return true - } - } - - return false -} - -// wrappedErrors is a workaround for wrapping multiple errors in environments -// where Go 1.20 is not available. It basically uses the already implemented -// functionality of joinedError to handle multiple errors with supplies a -// custom error message that is identical to the one we produce in Go 1.20 using -// multiple %w directives. -type wrappedErrors struct { - msg string - joinedError -} - -// Error returns the stored error string -func (we wrappedErrors) Error() string { - return we.msg -} - -// newError creates a new error message with a detailed error message. The -// message will be prefixed with the contents of the supplied error type. -// Additionally, more errors, that provide more context can be supplied which -// will be appended to the message. Since we cannot use of Go 1.20's possibility -// to include more than one %w formatting directive in [fmt.Errorf], we have to -// emulate that. -// -// For example, -// -// newError("no keyfunc was provided", ErrTokenUnverifiable) -// -// will produce the error string -// -// "token is unverifiable: no keyfunc was provided" -func newError(message string, err error, more ...error) error { - // We cannot wrap multiple errors here with %w, so we have to be a little - // bit creative. Basically, we are using %s instead of %w to produce the - // same error message and then throw the result into a custom error struct. - var format string - var args []any - if message != "" { - format = "%s: %s" - args = []any{err, message} - } else { - format = "%s" - args = []any{err} - } - errs := []error{err} - - for _, e := range more { - format += ": %s" - args = append(args, e) - errs = append(errs, e) - } - - err = &wrappedErrors{ - msg: fmt.Sprintf(format, args...), - joinedError: joinedError{errs: errs}, - } - return err -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/hmac.go b/vendor/github.com/golang-jwt/jwt/v5/hmac.go deleted file mode 100644 index aca600ce1..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/hmac.go +++ /dev/null @@ -1,104 +0,0 @@ -package jwt - -import ( - "crypto" - "crypto/hmac" - "errors" -) - -// SigningMethodHMAC implements the HMAC-SHA family of signing methods. -// Expects key type of []byte for both signing and validation -type SigningMethodHMAC struct { - Name string - Hash crypto.Hash -} - -// Specific instances for HS256 and company -var ( - SigningMethodHS256 *SigningMethodHMAC - SigningMethodHS384 *SigningMethodHMAC - SigningMethodHS512 *SigningMethodHMAC - ErrSignatureInvalid = errors.New("signature is invalid") -) - -func init() { - // HS256 - SigningMethodHS256 = &SigningMethodHMAC{"HS256", crypto.SHA256} - RegisterSigningMethod(SigningMethodHS256.Alg(), func() SigningMethod { - return SigningMethodHS256 - }) - - // HS384 - SigningMethodHS384 = &SigningMethodHMAC{"HS384", crypto.SHA384} - RegisterSigningMethod(SigningMethodHS384.Alg(), func() SigningMethod { - return SigningMethodHS384 - }) - - // HS512 - SigningMethodHS512 = &SigningMethodHMAC{"HS512", crypto.SHA512} - RegisterSigningMethod(SigningMethodHS512.Alg(), func() SigningMethod { - return SigningMethodHS512 - }) -} - -func (m *SigningMethodHMAC) Alg() string { - return m.Name -} - -// Verify implements token verification for the SigningMethod. Returns nil if -// the signature is valid. Key must be []byte. -// -// Note it is not advised to provide a []byte which was converted from a 'human -// readable' string using a subset of ASCII characters. To maximize entropy, you -// should ideally be providing a []byte key which was produced from a -// cryptographically random source, e.g. crypto/rand. Additional information -// about this, and why we intentionally are not supporting string as a key can -// be found on our usage guide -// https://golang-jwt.github.io/jwt/usage/signing_methods/#signing-methods-and-key-types. -func (m *SigningMethodHMAC) Verify(signingString string, sig []byte, key interface{}) error { - // Verify the key is the right type - keyBytes, ok := key.([]byte) - if !ok { - return newError("HMAC verify expects []byte", ErrInvalidKeyType) - } - - // Can we use the specified hashing method? - if !m.Hash.Available() { - return ErrHashUnavailable - } - - // This signing method is symmetric, so we validate the signature - // by reproducing the signature from the signing string and key, then - // comparing that against the provided signature. - hasher := hmac.New(m.Hash.New, keyBytes) - hasher.Write([]byte(signingString)) - if !hmac.Equal(sig, hasher.Sum(nil)) { - return ErrSignatureInvalid - } - - // No validation errors. Signature is good. - return nil -} - -// Sign implements token signing for the SigningMethod. Key must be []byte. -// -// Note it is not advised to provide a []byte which was converted from a 'human -// readable' string using a subset of ASCII characters. To maximize entropy, you -// should ideally be providing a []byte key which was produced from a -// cryptographically random source, e.g. crypto/rand. Additional information -// about this, and why we intentionally are not supporting string as a key can -// be found on our usage guide https://golang-jwt.github.io/jwt/usage/signing_methods/. -func (m *SigningMethodHMAC) Sign(signingString string, key interface{}) ([]byte, error) { - if keyBytes, ok := key.([]byte); ok { - if !m.Hash.Available() { - return nil, ErrHashUnavailable - } - - hasher := hmac.New(m.Hash.New, keyBytes) - hasher.Write([]byte(signingString)) - - return hasher.Sum(nil), nil - } - - return nil, newError("HMAC sign expects []byte", ErrInvalidKeyType) -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/map_claims.go b/vendor/github.com/golang-jwt/jwt/v5/map_claims.go deleted file mode 100644 index b2b51a1f8..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/map_claims.go +++ /dev/null @@ -1,109 +0,0 @@ -package jwt - -import ( - "encoding/json" - "fmt" -) - -// MapClaims is a claims type that uses the map[string]interface{} for JSON -// decoding. This is the default claims type if you don't supply one -type MapClaims map[string]interface{} - -// GetExpirationTime implements the Claims interface. -func (m MapClaims) GetExpirationTime() (*NumericDate, error) { - return m.parseNumericDate("exp") -} - -// GetNotBefore implements the Claims interface. -func (m MapClaims) GetNotBefore() (*NumericDate, error) { - return m.parseNumericDate("nbf") -} - -// GetIssuedAt implements the Claims interface. -func (m MapClaims) GetIssuedAt() (*NumericDate, error) { - return m.parseNumericDate("iat") -} - -// GetAudience implements the Claims interface. -func (m MapClaims) GetAudience() (ClaimStrings, error) { - return m.parseClaimsString("aud") -} - -// GetIssuer implements the Claims interface. -func (m MapClaims) GetIssuer() (string, error) { - return m.parseString("iss") -} - -// GetSubject implements the Claims interface. -func (m MapClaims) GetSubject() (string, error) { - return m.parseString("sub") -} - -// parseNumericDate tries to parse a key in the map claims type as a number -// date. This will succeed, if the underlying type is either a [float64] or a -// [json.Number]. Otherwise, nil will be returned. -func (m MapClaims) parseNumericDate(key string) (*NumericDate, error) { - v, ok := m[key] - if !ok { - return nil, nil - } - - switch exp := v.(type) { - case float64: - if exp == 0 { - return nil, nil - } - - return newNumericDateFromSeconds(exp), nil - case json.Number: - v, _ := exp.Float64() - - return newNumericDateFromSeconds(v), nil - } - - return nil, newError(fmt.Sprintf("%s is invalid", key), ErrInvalidType) -} - -// parseClaimsString tries to parse a key in the map claims type as a -// [ClaimsStrings] type, which can either be a string or an array of string. -func (m MapClaims) parseClaimsString(key string) (ClaimStrings, error) { - var cs []string - switch v := m[key].(type) { - case string: - cs = append(cs, v) - case []string: - cs = v - case []interface{}: - for _, a := range v { - vs, ok := a.(string) - if !ok { - return nil, newError(fmt.Sprintf("%s is invalid", key), ErrInvalidType) - } - cs = append(cs, vs) - } - } - - return cs, nil -} - -// parseString tries to parse a key in the map claims type as a [string] type. -// If the key does not exist, an empty string is returned. If the key has the -// wrong type, an error is returned. -func (m MapClaims) parseString(key string) (string, error) { - var ( - ok bool - raw interface{} - iss string - ) - raw, ok = m[key] - if !ok { - return "", nil - } - - iss, ok = raw.(string) - if !ok { - return "", newError(fmt.Sprintf("%s is invalid", key), ErrInvalidType) - } - - return iss, nil -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/none.go b/vendor/github.com/golang-jwt/jwt/v5/none.go deleted file mode 100644 index 685c2ea30..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/none.go +++ /dev/null @@ -1,50 +0,0 @@ -package jwt - -// SigningMethodNone implements the none signing method. This is required by the spec -// but you probably should never use it. -var SigningMethodNone *signingMethodNone - -const UnsafeAllowNoneSignatureType unsafeNoneMagicConstant = "none signing method allowed" - -var NoneSignatureTypeDisallowedError error - -type signingMethodNone struct{} -type unsafeNoneMagicConstant string - -func init() { - SigningMethodNone = &signingMethodNone{} - NoneSignatureTypeDisallowedError = newError("'none' signature type is not allowed", ErrTokenUnverifiable) - - RegisterSigningMethod(SigningMethodNone.Alg(), func() SigningMethod { - return SigningMethodNone - }) -} - -func (m *signingMethodNone) Alg() string { - return "none" -} - -// Only allow 'none' alg type if UnsafeAllowNoneSignatureType is specified as the key -func (m *signingMethodNone) Verify(signingString string, sig []byte, key interface{}) (err error) { - // Key must be UnsafeAllowNoneSignatureType to prevent accidentally - // accepting 'none' signing method - if _, ok := key.(unsafeNoneMagicConstant); !ok { - return NoneSignatureTypeDisallowedError - } - // If signing method is none, signature must be an empty string - if len(sig) != 0 { - return newError("'none' signing method with non-empty signature", ErrTokenUnverifiable) - } - - // Accept 'none' signing method. - return nil -} - -// Only allow 'none' signing if UnsafeAllowNoneSignatureType is specified as the key -func (m *signingMethodNone) Sign(signingString string, key interface{}) ([]byte, error) { - if _, ok := key.(unsafeNoneMagicConstant); ok { - return []byte{}, nil - } - - return nil, NoneSignatureTypeDisallowedError -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/parser.go b/vendor/github.com/golang-jwt/jwt/v5/parser.go deleted file mode 100644 index ecf99af78..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/parser.go +++ /dev/null @@ -1,238 +0,0 @@ -package jwt - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "fmt" - "strings" -) - -type Parser struct { - // If populated, only these methods will be considered valid. - validMethods []string - - // Use JSON Number format in JSON decoder. - useJSONNumber bool - - // Skip claims validation during token parsing. - skipClaimsValidation bool - - validator *Validator - - decodeStrict bool - - decodePaddingAllowed bool -} - -// NewParser creates a new Parser with the specified options -func NewParser(options ...ParserOption) *Parser { - p := &Parser{ - validator: &Validator{}, - } - - // Loop through our parsing options and apply them - for _, option := range options { - option(p) - } - - return p -} - -// Parse parses, validates, verifies the signature and returns the parsed token. -// keyFunc will receive the parsed token and should return the key for validating. -func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { - return p.ParseWithClaims(tokenString, MapClaims{}, keyFunc) -} - -// ParseWithClaims parses, validates, and verifies like Parse, but supplies a default object implementing the Claims -// interface. This provides default values which can be overridden and allows a caller to use their own type, rather -// than the default MapClaims implementation of Claims. -// -// Note: If you provide a custom claim implementation that embeds one of the standard claims (such as RegisteredClaims), -// make sure that a) you either embed a non-pointer version of the claims or b) if you are using a pointer, allocate the -// proper memory for it before passing in the overall claims, otherwise you might run into a panic. -func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) { - token, parts, err := p.ParseUnverified(tokenString, claims) - if err != nil { - return token, err - } - - // Verify signing method is in the required set - if p.validMethods != nil { - var signingMethodValid = false - var alg = token.Method.Alg() - for _, m := range p.validMethods { - if m == alg { - signingMethodValid = true - break - } - } - if !signingMethodValid { - // signing method is not in the listed set - return token, newError(fmt.Sprintf("signing method %v is invalid", alg), ErrTokenSignatureInvalid) - } - } - - // Decode signature - token.Signature, err = p.DecodeSegment(parts[2]) - if err != nil { - return token, newError("could not base64 decode signature", ErrTokenMalformed, err) - } - text := strings.Join(parts[0:2], ".") - - // Lookup key(s) - if keyFunc == nil { - // keyFunc was not provided. short circuiting validation - return token, newError("no keyfunc was provided", ErrTokenUnverifiable) - } - - got, err := keyFunc(token) - if err != nil { - return token, newError("error while executing keyfunc", ErrTokenUnverifiable, err) - } - - switch have := got.(type) { - case VerificationKeySet: - if len(have.Keys) == 0 { - return token, newError("keyfunc returned empty verification key set", ErrTokenUnverifiable) - } - // Iterate through keys and verify signature, skipping the rest when a match is found. - // Return the last error if no match is found. - for _, key := range have.Keys { - if err = token.Method.Verify(text, token.Signature, key); err == nil { - break - } - } - default: - err = token.Method.Verify(text, token.Signature, have) - } - if err != nil { - return token, newError("", ErrTokenSignatureInvalid, err) - } - - // Validate Claims - if !p.skipClaimsValidation { - // Make sure we have at least a default validator - if p.validator == nil { - p.validator = NewValidator() - } - - if err := p.validator.Validate(claims); err != nil { - return token, newError("", ErrTokenInvalidClaims, err) - } - } - - // No errors so far, token is valid. - token.Valid = true - - return token, nil -} - -// ParseUnverified parses the token but doesn't validate the signature. -// -// WARNING: Don't use this method unless you know what you're doing. -// -// It's only ever useful in cases where you know the signature is valid (since it has already -// been or will be checked elsewhere in the stack) and you want to extract values from it. -func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, newError("token contains an invalid number of segments", ErrTokenMalformed) - } - - token = &Token{Raw: tokenString} - - // parse Header - var headerBytes []byte - if headerBytes, err = p.DecodeSegment(parts[0]); err != nil { - return token, parts, newError("could not base64 decode header", ErrTokenMalformed, err) - } - if err = json.Unmarshal(headerBytes, &token.Header); err != nil { - return token, parts, newError("could not JSON decode header", ErrTokenMalformed, err) - } - - // parse Claims - token.Claims = claims - - claimBytes, err := p.DecodeSegment(parts[1]) - if err != nil { - return token, parts, newError("could not base64 decode claim", ErrTokenMalformed, err) - } - - // If `useJSONNumber` is enabled then we must use *json.Decoder to decode - // the claims. However, this comes with a performance penalty so only use - // it if we must and, otherwise, simple use json.Unmarshal. - if !p.useJSONNumber { - // JSON Unmarshal. Special case for map type to avoid weird pointer behavior. - if c, ok := token.Claims.(MapClaims); ok { - err = json.Unmarshal(claimBytes, &c) - } else { - err = json.Unmarshal(claimBytes, &claims) - } - } else { - dec := json.NewDecoder(bytes.NewBuffer(claimBytes)) - dec.UseNumber() - // JSON Decode. Special case for map type to avoid weird pointer behavior. - if c, ok := token.Claims.(MapClaims); ok { - err = dec.Decode(&c) - } else { - err = dec.Decode(&claims) - } - } - if err != nil { - return token, parts, newError("could not JSON decode claim", ErrTokenMalformed, err) - } - - // Lookup signature method - if method, ok := token.Header["alg"].(string); ok { - if token.Method = GetSigningMethod(method); token.Method == nil { - return token, parts, newError("signing method (alg) is unavailable", ErrTokenUnverifiable) - } - } else { - return token, parts, newError("signing method (alg) is unspecified", ErrTokenUnverifiable) - } - - return token, parts, nil -} - -// DecodeSegment decodes a JWT specific base64url encoding. This function will -// take into account whether the [Parser] is configured with additional options, -// such as [WithStrictDecoding] or [WithPaddingAllowed]. -func (p *Parser) DecodeSegment(seg string) ([]byte, error) { - encoding := base64.RawURLEncoding - - if p.decodePaddingAllowed { - if l := len(seg) % 4; l > 0 { - seg += strings.Repeat("=", 4-l) - } - encoding = base64.URLEncoding - } - - if p.decodeStrict { - encoding = encoding.Strict() - } - return encoding.DecodeString(seg) -} - -// Parse parses, validates, verifies the signature and returns the parsed token. -// keyFunc will receive the parsed token and should return the cryptographic key -// for verifying the signature. The caller is strongly encouraged to set the -// WithValidMethods option to validate the 'alg' claim in the token matches the -// expected algorithm. For more details about the importance of validating the -// 'alg' claim, see -// https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/ -func Parse(tokenString string, keyFunc Keyfunc, options ...ParserOption) (*Token, error) { - return NewParser(options...).Parse(tokenString, keyFunc) -} - -// ParseWithClaims is a shortcut for NewParser().ParseWithClaims(). -// -// Note: If you provide a custom claim implementation that embeds one of the -// standard claims (such as RegisteredClaims), make sure that a) you either -// embed a non-pointer version of the claims or b) if you are using a pointer, -// allocate the proper memory for it before passing in the overall claims, -// otherwise you might run into a panic. -func ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc, options ...ParserOption) (*Token, error) { - return NewParser(options...).ParseWithClaims(tokenString, claims, keyFunc) -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/parser_option.go b/vendor/github.com/golang-jwt/jwt/v5/parser_option.go deleted file mode 100644 index 88a780fbd..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/parser_option.go +++ /dev/null @@ -1,128 +0,0 @@ -package jwt - -import "time" - -// ParserOption is used to implement functional-style options that modify the -// behavior of the parser. To add new options, just create a function (ideally -// beginning with With or Without) that returns an anonymous function that takes -// a *Parser type as input and manipulates its configuration accordingly. -type ParserOption func(*Parser) - -// WithValidMethods is an option to supply algorithm methods that the parser -// will check. Only those methods will be considered valid. It is heavily -// encouraged to use this option in order to prevent attacks such as -// https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/. -func WithValidMethods(methods []string) ParserOption { - return func(p *Parser) { - p.validMethods = methods - } -} - -// WithJSONNumber is an option to configure the underlying JSON parser with -// UseNumber. -func WithJSONNumber() ParserOption { - return func(p *Parser) { - p.useJSONNumber = true - } -} - -// WithoutClaimsValidation is an option to disable claims validation. This -// option should only be used if you exactly know what you are doing. -func WithoutClaimsValidation() ParserOption { - return func(p *Parser) { - p.skipClaimsValidation = true - } -} - -// WithLeeway returns the ParserOption for specifying the leeway window. -func WithLeeway(leeway time.Duration) ParserOption { - return func(p *Parser) { - p.validator.leeway = leeway - } -} - -// WithTimeFunc returns the ParserOption for specifying the time func. The -// primary use-case for this is testing. If you are looking for a way to account -// for clock-skew, WithLeeway should be used instead. -func WithTimeFunc(f func() time.Time) ParserOption { - return func(p *Parser) { - p.validator.timeFunc = f - } -} - -// WithIssuedAt returns the ParserOption to enable verification -// of issued-at. -func WithIssuedAt() ParserOption { - return func(p *Parser) { - p.validator.verifyIat = true - } -} - -// WithExpirationRequired returns the ParserOption to make exp claim required. -// By default exp claim is optional. -func WithExpirationRequired() ParserOption { - return func(p *Parser) { - p.validator.requireExp = true - } -} - -// WithAudience configures the validator to require the specified audience in -// the `aud` claim. Validation will fail if the audience is not listed in the -// token or the `aud` claim is missing. -// -// NOTE: While the `aud` claim is OPTIONAL in a JWT, the handling of it is -// application-specific. Since this validation API is helping developers in -// writing secure application, we decided to REQUIRE the existence of the claim, -// if an audience is expected. -func WithAudience(aud string) ParserOption { - return func(p *Parser) { - p.validator.expectedAud = aud - } -} - -// WithIssuer configures the validator to require the specified issuer in the -// `iss` claim. Validation will fail if a different issuer is specified in the -// token or the `iss` claim is missing. -// -// NOTE: While the `iss` claim is OPTIONAL in a JWT, the handling of it is -// application-specific. Since this validation API is helping developers in -// writing secure application, we decided to REQUIRE the existence of the claim, -// if an issuer is expected. -func WithIssuer(iss string) ParserOption { - return func(p *Parser) { - p.validator.expectedIss = iss - } -} - -// WithSubject configures the validator to require the specified subject in the -// `sub` claim. Validation will fail if a different subject is specified in the -// token or the `sub` claim is missing. -// -// NOTE: While the `sub` claim is OPTIONAL in a JWT, the handling of it is -// application-specific. Since this validation API is helping developers in -// writing secure application, we decided to REQUIRE the existence of the claim, -// if a subject is expected. -func WithSubject(sub string) ParserOption { - return func(p *Parser) { - p.validator.expectedSub = sub - } -} - -// WithPaddingAllowed will enable the codec used for decoding JWTs to allow -// padding. Note that the JWS RFC7515 states that the tokens will utilize a -// Base64url encoding with no padding. Unfortunately, some implementations of -// JWT are producing non-standard tokens, and thus require support for decoding. -func WithPaddingAllowed() ParserOption { - return func(p *Parser) { - p.decodePaddingAllowed = true - } -} - -// WithStrictDecoding will switch the codec used for decoding JWTs into strict -// mode. In this mode, the decoder requires that trailing padding bits are zero, -// as described in RFC 4648 section 3.5. -func WithStrictDecoding() ParserOption { - return func(p *Parser) { - p.decodeStrict = true - } -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/registered_claims.go b/vendor/github.com/golang-jwt/jwt/v5/registered_claims.go deleted file mode 100644 index 77951a531..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/registered_claims.go +++ /dev/null @@ -1,63 +0,0 @@ -package jwt - -// RegisteredClaims are a structured version of the JWT Claims Set, -// restricted to Registered Claim Names, as referenced at -// https://datatracker.ietf.org/doc/html/rfc7519#section-4.1 -// -// This type can be used on its own, but then additional private and -// public claims embedded in the JWT will not be parsed. The typical use-case -// therefore is to embedded this in a user-defined claim type. -// -// See examples for how to use this with your own claim types. -type RegisteredClaims struct { - // the `iss` (Issuer) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.1 - Issuer string `json:"iss,omitempty"` - - // the `sub` (Subject) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.2 - Subject string `json:"sub,omitempty"` - - // the `aud` (Audience) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.3 - Audience ClaimStrings `json:"aud,omitempty"` - - // the `exp` (Expiration Time) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.4 - ExpiresAt *NumericDate `json:"exp,omitempty"` - - // the `nbf` (Not Before) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.5 - NotBefore *NumericDate `json:"nbf,omitempty"` - - // the `iat` (Issued At) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.6 - IssuedAt *NumericDate `json:"iat,omitempty"` - - // the `jti` (JWT ID) claim. See https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.7 - ID string `json:"jti,omitempty"` -} - -// GetExpirationTime implements the Claims interface. -func (c RegisteredClaims) GetExpirationTime() (*NumericDate, error) { - return c.ExpiresAt, nil -} - -// GetNotBefore implements the Claims interface. -func (c RegisteredClaims) GetNotBefore() (*NumericDate, error) { - return c.NotBefore, nil -} - -// GetIssuedAt implements the Claims interface. -func (c RegisteredClaims) GetIssuedAt() (*NumericDate, error) { - return c.IssuedAt, nil -} - -// GetAudience implements the Claims interface. -func (c RegisteredClaims) GetAudience() (ClaimStrings, error) { - return c.Audience, nil -} - -// GetIssuer implements the Claims interface. -func (c RegisteredClaims) GetIssuer() (string, error) { - return c.Issuer, nil -} - -// GetSubject implements the Claims interface. -func (c RegisteredClaims) GetSubject() (string, error) { - return c.Subject, nil -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa.go b/vendor/github.com/golang-jwt/jwt/v5/rsa.go deleted file mode 100644 index 83cbee6ae..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/rsa.go +++ /dev/null @@ -1,93 +0,0 @@ -package jwt - -import ( - "crypto" - "crypto/rand" - "crypto/rsa" -) - -// SigningMethodRSA implements the RSA family of signing methods. -// Expects *rsa.PrivateKey for signing and *rsa.PublicKey for validation -type SigningMethodRSA struct { - Name string - Hash crypto.Hash -} - -// Specific instances for RS256 and company -var ( - SigningMethodRS256 *SigningMethodRSA - SigningMethodRS384 *SigningMethodRSA - SigningMethodRS512 *SigningMethodRSA -) - -func init() { - // RS256 - SigningMethodRS256 = &SigningMethodRSA{"RS256", crypto.SHA256} - RegisterSigningMethod(SigningMethodRS256.Alg(), func() SigningMethod { - return SigningMethodRS256 - }) - - // RS384 - SigningMethodRS384 = &SigningMethodRSA{"RS384", crypto.SHA384} - RegisterSigningMethod(SigningMethodRS384.Alg(), func() SigningMethod { - return SigningMethodRS384 - }) - - // RS512 - SigningMethodRS512 = &SigningMethodRSA{"RS512", crypto.SHA512} - RegisterSigningMethod(SigningMethodRS512.Alg(), func() SigningMethod { - return SigningMethodRS512 - }) -} - -func (m *SigningMethodRSA) Alg() string { - return m.Name -} - -// Verify implements token verification for the SigningMethod -// For this signing method, must be an *rsa.PublicKey structure. -func (m *SigningMethodRSA) Verify(signingString string, sig []byte, key interface{}) error { - var rsaKey *rsa.PublicKey - var ok bool - - if rsaKey, ok = key.(*rsa.PublicKey); !ok { - return newError("RSA verify expects *rsa.PublicKey", ErrInvalidKeyType) - } - - // Create hasher - if !m.Hash.Available() { - return ErrHashUnavailable - } - hasher := m.Hash.New() - hasher.Write([]byte(signingString)) - - // Verify the signature - return rsa.VerifyPKCS1v15(rsaKey, m.Hash, hasher.Sum(nil), sig) -} - -// Sign implements token signing for the SigningMethod -// For this signing method, must be an *rsa.PrivateKey structure. -func (m *SigningMethodRSA) Sign(signingString string, key interface{}) ([]byte, error) { - var rsaKey *rsa.PrivateKey - var ok bool - - // Validate type of key - if rsaKey, ok = key.(*rsa.PrivateKey); !ok { - return nil, newError("RSA sign expects *rsa.PrivateKey", ErrInvalidKeyType) - } - - // Create the hasher - if !m.Hash.Available() { - return nil, ErrHashUnavailable - } - - hasher := m.Hash.New() - hasher.Write([]byte(signingString)) - - // Sign the string and return the encoded bytes - if sigBytes, err := rsa.SignPKCS1v15(rand.Reader, rsaKey, m.Hash, hasher.Sum(nil)); err == nil { - return sigBytes, nil - } else { - return nil, err - } -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go b/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go deleted file mode 100644 index 28c386ec4..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/rsa_pss.go +++ /dev/null @@ -1,135 +0,0 @@ -//go:build go1.4 -// +build go1.4 - -package jwt - -import ( - "crypto" - "crypto/rand" - "crypto/rsa" -) - -// SigningMethodRSAPSS implements the RSAPSS family of signing methods signing methods -type SigningMethodRSAPSS struct { - *SigningMethodRSA - Options *rsa.PSSOptions - // VerifyOptions is optional. If set overrides Options for rsa.VerifyPPS. - // Used to accept tokens signed with rsa.PSSSaltLengthAuto, what doesn't follow - // https://tools.ietf.org/html/rfc7518#section-3.5 but was used previously. - // See https://github.com/dgrijalva/jwt-go/issues/285#issuecomment-437451244 for details. - VerifyOptions *rsa.PSSOptions -} - -// Specific instances for RS/PS and company. -var ( - SigningMethodPS256 *SigningMethodRSAPSS - SigningMethodPS384 *SigningMethodRSAPSS - SigningMethodPS512 *SigningMethodRSAPSS -) - -func init() { - // PS256 - SigningMethodPS256 = &SigningMethodRSAPSS{ - SigningMethodRSA: &SigningMethodRSA{ - Name: "PS256", - Hash: crypto.SHA256, - }, - Options: &rsa.PSSOptions{ - SaltLength: rsa.PSSSaltLengthEqualsHash, - }, - VerifyOptions: &rsa.PSSOptions{ - SaltLength: rsa.PSSSaltLengthAuto, - }, - } - RegisterSigningMethod(SigningMethodPS256.Alg(), func() SigningMethod { - return SigningMethodPS256 - }) - - // PS384 - SigningMethodPS384 = &SigningMethodRSAPSS{ - SigningMethodRSA: &SigningMethodRSA{ - Name: "PS384", - Hash: crypto.SHA384, - }, - Options: &rsa.PSSOptions{ - SaltLength: rsa.PSSSaltLengthEqualsHash, - }, - VerifyOptions: &rsa.PSSOptions{ - SaltLength: rsa.PSSSaltLengthAuto, - }, - } - RegisterSigningMethod(SigningMethodPS384.Alg(), func() SigningMethod { - return SigningMethodPS384 - }) - - // PS512 - SigningMethodPS512 = &SigningMethodRSAPSS{ - SigningMethodRSA: &SigningMethodRSA{ - Name: "PS512", - Hash: crypto.SHA512, - }, - Options: &rsa.PSSOptions{ - SaltLength: rsa.PSSSaltLengthEqualsHash, - }, - VerifyOptions: &rsa.PSSOptions{ - SaltLength: rsa.PSSSaltLengthAuto, - }, - } - RegisterSigningMethod(SigningMethodPS512.Alg(), func() SigningMethod { - return SigningMethodPS512 - }) -} - -// Verify implements token verification for the SigningMethod. -// For this verify method, key must be an rsa.PublicKey struct -func (m *SigningMethodRSAPSS) Verify(signingString string, sig []byte, key interface{}) error { - var rsaKey *rsa.PublicKey - switch k := key.(type) { - case *rsa.PublicKey: - rsaKey = k - default: - return newError("RSA-PSS verify expects *rsa.PublicKey", ErrInvalidKeyType) - } - - // Create hasher - if !m.Hash.Available() { - return ErrHashUnavailable - } - hasher := m.Hash.New() - hasher.Write([]byte(signingString)) - - opts := m.Options - if m.VerifyOptions != nil { - opts = m.VerifyOptions - } - - return rsa.VerifyPSS(rsaKey, m.Hash, hasher.Sum(nil), sig, opts) -} - -// Sign implements token signing for the SigningMethod. -// For this signing method, key must be an rsa.PrivateKey struct -func (m *SigningMethodRSAPSS) Sign(signingString string, key interface{}) ([]byte, error) { - var rsaKey *rsa.PrivateKey - - switch k := key.(type) { - case *rsa.PrivateKey: - rsaKey = k - default: - return nil, newError("RSA-PSS sign expects *rsa.PrivateKey", ErrInvalidKeyType) - } - - // Create the hasher - if !m.Hash.Available() { - return nil, ErrHashUnavailable - } - - hasher := m.Hash.New() - hasher.Write([]byte(signingString)) - - // Sign the string and return the encoded bytes - if sigBytes, err := rsa.SignPSS(rand.Reader, rsaKey, m.Hash, hasher.Sum(nil), m.Options); err == nil { - return sigBytes, nil - } else { - return nil, err - } -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go b/vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go deleted file mode 100644 index b3aeebbe1..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/rsa_utils.go +++ /dev/null @@ -1,107 +0,0 @@ -package jwt - -import ( - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "errors" -) - -var ( - ErrKeyMustBePEMEncoded = errors.New("invalid key: Key must be a PEM encoded PKCS1 or PKCS8 key") - ErrNotRSAPrivateKey = errors.New("key is not a valid RSA private key") - ErrNotRSAPublicKey = errors.New("key is not a valid RSA public key") -) - -// ParseRSAPrivateKeyFromPEM parses a PEM encoded PKCS1 or PKCS8 private key -func ParseRSAPrivateKeyFromPEM(key []byte) (*rsa.PrivateKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - var parsedKey interface{} - if parsedKey, err = x509.ParsePKCS1PrivateKey(block.Bytes); err != nil { - if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { - return nil, err - } - } - - var pkey *rsa.PrivateKey - var ok bool - if pkey, ok = parsedKey.(*rsa.PrivateKey); !ok { - return nil, ErrNotRSAPrivateKey - } - - return pkey, nil -} - -// ParseRSAPrivateKeyFromPEMWithPassword parses a PEM encoded PKCS1 or PKCS8 private key protected with password -// -// Deprecated: This function is deprecated and should not be used anymore. It uses the deprecated x509.DecryptPEMBlock -// function, which was deprecated since RFC 1423 is regarded insecure by design. Unfortunately, there is no alternative -// in the Go standard library for now. See https://github.com/golang/go/issues/8860. -func ParseRSAPrivateKeyFromPEMWithPassword(key []byte, password string) (*rsa.PrivateKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - var parsedKey interface{} - - var blockDecrypted []byte - if blockDecrypted, err = x509.DecryptPEMBlock(block, []byte(password)); err != nil { - return nil, err - } - - if parsedKey, err = x509.ParsePKCS1PrivateKey(blockDecrypted); err != nil { - if parsedKey, err = x509.ParsePKCS8PrivateKey(blockDecrypted); err != nil { - return nil, err - } - } - - var pkey *rsa.PrivateKey - var ok bool - if pkey, ok = parsedKey.(*rsa.PrivateKey); !ok { - return nil, ErrNotRSAPrivateKey - } - - return pkey, nil -} - -// ParseRSAPublicKeyFromPEM parses a certificate or a PEM encoded PKCS1 or PKIX public key -func ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error) { - var err error - - // Parse PEM block - var block *pem.Block - if block, _ = pem.Decode(key); block == nil { - return nil, ErrKeyMustBePEMEncoded - } - - // Parse the key - var parsedKey interface{} - if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { - if cert, err := x509.ParseCertificate(block.Bytes); err == nil { - parsedKey = cert.PublicKey - } else { - if parsedKey, err = x509.ParsePKCS1PublicKey(block.Bytes); err != nil { - return nil, err - } - } - } - - var pkey *rsa.PublicKey - var ok bool - if pkey, ok = parsedKey.(*rsa.PublicKey); !ok { - return nil, ErrNotRSAPublicKey - } - - return pkey, nil -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/signing_method.go b/vendor/github.com/golang-jwt/jwt/v5/signing_method.go deleted file mode 100644 index 0d73631c1..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/signing_method.go +++ /dev/null @@ -1,49 +0,0 @@ -package jwt - -import ( - "sync" -) - -var signingMethods = map[string]func() SigningMethod{} -var signingMethodLock = new(sync.RWMutex) - -// SigningMethod can be used add new methods for signing or verifying tokens. It -// takes a decoded signature as an input in the Verify function and produces a -// signature in Sign. The signature is then usually base64 encoded as part of a -// JWT. -type SigningMethod interface { - Verify(signingString string, sig []byte, key interface{}) error // Returns nil if signature is valid - Sign(signingString string, key interface{}) ([]byte, error) // Returns signature or error - Alg() string // returns the alg identifier for this method (example: 'HS256') -} - -// RegisterSigningMethod registers the "alg" name and a factory function for signing method. -// This is typically done during init() in the method's implementation -func RegisterSigningMethod(alg string, f func() SigningMethod) { - signingMethodLock.Lock() - defer signingMethodLock.Unlock() - - signingMethods[alg] = f -} - -// GetSigningMethod retrieves a signing method from an "alg" string -func GetSigningMethod(alg string) (method SigningMethod) { - signingMethodLock.RLock() - defer signingMethodLock.RUnlock() - - if methodF, ok := signingMethods[alg]; ok { - method = methodF() - } - return -} - -// GetAlgorithms returns a list of registered "alg" names -func GetAlgorithms() (algs []string) { - signingMethodLock.RLock() - defer signingMethodLock.RUnlock() - - for alg := range signingMethods { - algs = append(algs, alg) - } - return -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf b/vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf deleted file mode 100644 index 53745d51d..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/staticcheck.conf +++ /dev/null @@ -1 +0,0 @@ -checks = ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1023"] diff --git a/vendor/github.com/golang-jwt/jwt/v5/token.go b/vendor/github.com/golang-jwt/jwt/v5/token.go deleted file mode 100644 index 352873a2d..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/token.go +++ /dev/null @@ -1,100 +0,0 @@ -package jwt - -import ( - "crypto" - "encoding/base64" - "encoding/json" -) - -// Keyfunc will be used by the Parse methods as a callback function to supply -// the key for verification. The function receives the parsed, but unverified -// Token. This allows you to use properties in the Header of the token (such as -// `kid`) to identify which key to use. -// -// The returned interface{} may be a single key or a VerificationKeySet containing -// multiple keys. -type Keyfunc func(*Token) (interface{}, error) - -// VerificationKey represents a public or secret key for verifying a token's signature. -type VerificationKey interface { - crypto.PublicKey | []uint8 -} - -// VerificationKeySet is a set of public or secret keys. It is used by the parser to verify a token. -type VerificationKeySet struct { - Keys []VerificationKey -} - -// Token represents a JWT Token. Different fields will be used depending on -// whether you're creating or parsing/verifying a token. -type Token struct { - Raw string // Raw contains the raw token. Populated when you [Parse] a token - Method SigningMethod // Method is the signing method used or to be used - Header map[string]interface{} // Header is the first segment of the token in decoded form - Claims Claims // Claims is the second segment of the token in decoded form - Signature []byte // Signature is the third segment of the token in decoded form. Populated when you Parse a token - Valid bool // Valid specifies if the token is valid. Populated when you Parse/Verify a token -} - -// New creates a new [Token] with the specified signing method and an empty map -// of claims. Additional options can be specified, but are currently unused. -func New(method SigningMethod, opts ...TokenOption) *Token { - return NewWithClaims(method, MapClaims{}, opts...) -} - -// NewWithClaims creates a new [Token] with the specified signing method and -// claims. Additional options can be specified, but are currently unused. -func NewWithClaims(method SigningMethod, claims Claims, opts ...TokenOption) *Token { - return &Token{ - Header: map[string]interface{}{ - "typ": "JWT", - "alg": method.Alg(), - }, - Claims: claims, - Method: method, - } -} - -// SignedString creates and returns a complete, signed JWT. The token is signed -// using the SigningMethod specified in the token. Please refer to -// https://golang-jwt.github.io/jwt/usage/signing_methods/#signing-methods-and-key-types -// for an overview of the different signing methods and their respective key -// types. -func (t *Token) SignedString(key interface{}) (string, error) { - sstr, err := t.SigningString() - if err != nil { - return "", err - } - - sig, err := t.Method.Sign(sstr, key) - if err != nil { - return "", err - } - - return sstr + "." + t.EncodeSegment(sig), nil -} - -// SigningString generates the signing string. This is the most expensive part -// of the whole deal. Unless you need this for something special, just go -// straight for the SignedString. -func (t *Token) SigningString() (string, error) { - h, err := json.Marshal(t.Header) - if err != nil { - return "", err - } - - c, err := json.Marshal(t.Claims) - if err != nil { - return "", err - } - - return t.EncodeSegment(h) + "." + t.EncodeSegment(c), nil -} - -// EncodeSegment encodes a JWT specific base64url encoding with padding -// stripped. In the future, this function might take into account a -// [TokenOption]. Therefore, this function exists as a method of [Token], rather -// than a global function. -func (*Token) EncodeSegment(seg []byte) string { - return base64.RawURLEncoding.EncodeToString(seg) -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/token_option.go b/vendor/github.com/golang-jwt/jwt/v5/token_option.go deleted file mode 100644 index b4ae3badf..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/token_option.go +++ /dev/null @@ -1,5 +0,0 @@ -package jwt - -// TokenOption is a reserved type, which provides some forward compatibility, -// if we ever want to introduce token creation-related options. -type TokenOption func(*Token) diff --git a/vendor/github.com/golang-jwt/jwt/v5/types.go b/vendor/github.com/golang-jwt/jwt/v5/types.go deleted file mode 100644 index b2655a9e6..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/types.go +++ /dev/null @@ -1,149 +0,0 @@ -package jwt - -import ( - "encoding/json" - "fmt" - "math" - "strconv" - "time" -) - -// TimePrecision sets the precision of times and dates within this library. This -// has an influence on the precision of times when comparing expiry or other -// related time fields. Furthermore, it is also the precision of times when -// serializing. -// -// For backwards compatibility the default precision is set to seconds, so that -// no fractional timestamps are generated. -var TimePrecision = time.Second - -// MarshalSingleStringAsArray modifies the behavior of the ClaimStrings type, -// especially its MarshalJSON function. -// -// If it is set to true (the default), it will always serialize the type as an -// array of strings, even if it just contains one element, defaulting to the -// behavior of the underlying []string. If it is set to false, it will serialize -// to a single string, if it contains one element. Otherwise, it will serialize -// to an array of strings. -var MarshalSingleStringAsArray = true - -// NumericDate represents a JSON numeric date value, as referenced at -// https://datatracker.ietf.org/doc/html/rfc7519#section-2. -type NumericDate struct { - time.Time -} - -// NewNumericDate constructs a new *NumericDate from a standard library time.Time struct. -// It will truncate the timestamp according to the precision specified in TimePrecision. -func NewNumericDate(t time.Time) *NumericDate { - return &NumericDate{t.Truncate(TimePrecision)} -} - -// newNumericDateFromSeconds creates a new *NumericDate out of a float64 representing a -// UNIX epoch with the float fraction representing non-integer seconds. -func newNumericDateFromSeconds(f float64) *NumericDate { - round, frac := math.Modf(f) - return NewNumericDate(time.Unix(int64(round), int64(frac*1e9))) -} - -// MarshalJSON is an implementation of the json.RawMessage interface and serializes the UNIX epoch -// represented in NumericDate to a byte array, using the precision specified in TimePrecision. -func (date NumericDate) MarshalJSON() (b []byte, err error) { - var prec int - if TimePrecision < time.Second { - prec = int(math.Log10(float64(time.Second) / float64(TimePrecision))) - } - truncatedDate := date.Truncate(TimePrecision) - - // For very large timestamps, UnixNano would overflow an int64, but this - // function requires nanosecond level precision, so we have to use the - // following technique to get round the issue: - // - // 1. Take the normal unix timestamp to form the whole number part of the - // output, - // 2. Take the result of the Nanosecond function, which returns the offset - // within the second of the particular unix time instance, to form the - // decimal part of the output - // 3. Concatenate them to produce the final result - seconds := strconv.FormatInt(truncatedDate.Unix(), 10) - nanosecondsOffset := strconv.FormatFloat(float64(truncatedDate.Nanosecond())/float64(time.Second), 'f', prec, 64) - - output := append([]byte(seconds), []byte(nanosecondsOffset)[1:]...) - - return output, nil -} - -// UnmarshalJSON is an implementation of the json.RawMessage interface and -// deserializes a [NumericDate] from a JSON representation, i.e. a -// [json.Number]. This number represents an UNIX epoch with either integer or -// non-integer seconds. -func (date *NumericDate) UnmarshalJSON(b []byte) (err error) { - var ( - number json.Number - f float64 - ) - - if err = json.Unmarshal(b, &number); err != nil { - return fmt.Errorf("could not parse NumericData: %w", err) - } - - if f, err = number.Float64(); err != nil { - return fmt.Errorf("could not convert json number value to float: %w", err) - } - - n := newNumericDateFromSeconds(f) - *date = *n - - return nil -} - -// ClaimStrings is basically just a slice of strings, but it can be either -// serialized from a string array or just a string. This type is necessary, -// since the "aud" claim can either be a single string or an array. -type ClaimStrings []string - -func (s *ClaimStrings) UnmarshalJSON(data []byte) (err error) { - var value interface{} - - if err = json.Unmarshal(data, &value); err != nil { - return err - } - - var aud []string - - switch v := value.(type) { - case string: - aud = append(aud, v) - case []string: - aud = ClaimStrings(v) - case []interface{}: - for _, vv := range v { - vs, ok := vv.(string) - if !ok { - return ErrInvalidType - } - aud = append(aud, vs) - } - case nil: - return nil - default: - return ErrInvalidType - } - - *s = aud - - return -} - -func (s ClaimStrings) MarshalJSON() (b []byte, err error) { - // This handles a special case in the JWT RFC. If the string array, e.g. - // used by the "aud" field, only contains one element, it MAY be serialized - // as a single string. This may or may not be desired based on the ecosystem - // of other JWT library used, so we make it configurable by the variable - // MarshalSingleStringAsArray. - if len(s) == 1 && !MarshalSingleStringAsArray { - return json.Marshal(s[0]) - } - - return json.Marshal([]string(s)) -} diff --git a/vendor/github.com/golang-jwt/jwt/v5/validator.go b/vendor/github.com/golang-jwt/jwt/v5/validator.go deleted file mode 100644 index 008ecd871..000000000 --- a/vendor/github.com/golang-jwt/jwt/v5/validator.go +++ /dev/null @@ -1,316 +0,0 @@ -package jwt - -import ( - "crypto/subtle" - "fmt" - "time" -) - -// ClaimsValidator is an interface that can be implemented by custom claims who -// wish to execute any additional claims validation based on -// application-specific logic. The Validate function is then executed in -// addition to the regular claims validation and any error returned is appended -// to the final validation result. -// -// type MyCustomClaims struct { -// Foo string `json:"foo"` -// jwt.RegisteredClaims -// } -// -// func (m MyCustomClaims) Validate() error { -// if m.Foo != "bar" { -// return errors.New("must be foobar") -// } -// return nil -// } -type ClaimsValidator interface { - Claims - Validate() error -} - -// Validator is the core of the new Validation API. It is automatically used by -// a [Parser] during parsing and can be modified with various parser options. -// -// The [NewValidator] function should be used to create an instance of this -// struct. -type Validator struct { - // leeway is an optional leeway that can be provided to account for clock skew. - leeway time.Duration - - // timeFunc is used to supply the current time that is needed for - // validation. If unspecified, this defaults to time.Now. - timeFunc func() time.Time - - // requireExp specifies whether the exp claim is required - requireExp bool - - // verifyIat specifies whether the iat (Issued At) claim will be verified. - // According to https://www.rfc-editor.org/rfc/rfc7519#section-4.1.6 this - // only specifies the age of the token, but no validation check is - // necessary. However, if wanted, it can be checked if the iat is - // unrealistic, i.e., in the future. - verifyIat bool - - // expectedAud contains the audience this token expects. Supplying an empty - // string will disable aud checking. - expectedAud string - - // expectedIss contains the issuer this token expects. Supplying an empty - // string will disable iss checking. - expectedIss string - - // expectedSub contains the subject this token expects. Supplying an empty - // string will disable sub checking. - expectedSub string -} - -// NewValidator can be used to create a stand-alone validator with the supplied -// options. This validator can then be used to validate already parsed claims. -// -// Note: Under normal circumstances, explicitly creating a validator is not -// needed and can potentially be dangerous; instead functions of the [Parser] -// class should be used. -// -// The [Validator] is only checking the *validity* of the claims, such as its -// expiration time, but it does NOT perform *signature verification* of the -// token. -func NewValidator(opts ...ParserOption) *Validator { - p := NewParser(opts...) - return p.validator -} - -// Validate validates the given claims. It will also perform any custom -// validation if claims implements the [ClaimsValidator] interface. -// -// Note: It will NOT perform any *signature verification* on the token that -// contains the claims and expects that the [Claim] was already successfully -// verified. -func (v *Validator) Validate(claims Claims) error { - var ( - now time.Time - errs []error = make([]error, 0, 6) - err error - ) - - // Check, if we have a time func - if v.timeFunc != nil { - now = v.timeFunc() - } else { - now = time.Now() - } - - // We always need to check the expiration time, but usage of the claim - // itself is OPTIONAL by default. requireExp overrides this behavior - // and makes the exp claim mandatory. - if err = v.verifyExpiresAt(claims, now, v.requireExp); err != nil { - errs = append(errs, err) - } - - // We always need to check not-before, but usage of the claim itself is - // OPTIONAL. - if err = v.verifyNotBefore(claims, now, false); err != nil { - errs = append(errs, err) - } - - // Check issued-at if the option is enabled - if v.verifyIat { - if err = v.verifyIssuedAt(claims, now, false); err != nil { - errs = append(errs, err) - } - } - - // If we have an expected audience, we also require the audience claim - if v.expectedAud != "" { - if err = v.verifyAudience(claims, v.expectedAud, true); err != nil { - errs = append(errs, err) - } - } - - // If we have an expected issuer, we also require the issuer claim - if v.expectedIss != "" { - if err = v.verifyIssuer(claims, v.expectedIss, true); err != nil { - errs = append(errs, err) - } - } - - // If we have an expected subject, we also require the subject claim - if v.expectedSub != "" { - if err = v.verifySubject(claims, v.expectedSub, true); err != nil { - errs = append(errs, err) - } - } - - // Finally, we want to give the claim itself some possibility to do some - // additional custom validation based on a custom Validate function. - cvt, ok := claims.(ClaimsValidator) - if ok { - if err := cvt.Validate(); err != nil { - errs = append(errs, err) - } - } - - if len(errs) == 0 { - return nil - } - - return joinErrors(errs...) -} - -// verifyExpiresAt compares the exp claim in claims against cmp. This function -// will succeed if cmp < exp. Additional leeway is taken into account. -// -// If exp is not set, it will succeed if the claim is not required, -// otherwise ErrTokenRequiredClaimMissing will be returned. -// -// Additionally, if any error occurs while retrieving the claim, e.g., when its -// the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *Validator) verifyExpiresAt(claims Claims, cmp time.Time, required bool) error { - exp, err := claims.GetExpirationTime() - if err != nil { - return err - } - - if exp == nil { - return errorIfRequired(required, "exp") - } - - return errorIfFalse(cmp.Before((exp.Time).Add(+v.leeway)), ErrTokenExpired) -} - -// verifyIssuedAt compares the iat claim in claims against cmp. This function -// will succeed if cmp >= iat. Additional leeway is taken into account. -// -// If iat is not set, it will succeed if the claim is not required, -// otherwise ErrTokenRequiredClaimMissing will be returned. -// -// Additionally, if any error occurs while retrieving the claim, e.g., when its -// the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *Validator) verifyIssuedAt(claims Claims, cmp time.Time, required bool) error { - iat, err := claims.GetIssuedAt() - if err != nil { - return err - } - - if iat == nil { - return errorIfRequired(required, "iat") - } - - return errorIfFalse(!cmp.Before(iat.Add(-v.leeway)), ErrTokenUsedBeforeIssued) -} - -// verifyNotBefore compares the nbf claim in claims against cmp. This function -// will return true if cmp >= nbf. Additional leeway is taken into account. -// -// If nbf is not set, it will succeed if the claim is not required, -// otherwise ErrTokenRequiredClaimMissing will be returned. -// -// Additionally, if any error occurs while retrieving the claim, e.g., when its -// the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *Validator) verifyNotBefore(claims Claims, cmp time.Time, required bool) error { - nbf, err := claims.GetNotBefore() - if err != nil { - return err - } - - if nbf == nil { - return errorIfRequired(required, "nbf") - } - - return errorIfFalse(!cmp.Before(nbf.Add(-v.leeway)), ErrTokenNotValidYet) -} - -// verifyAudience compares the aud claim against cmp. -// -// If aud is not set or an empty list, it will succeed if the claim is not required, -// otherwise ErrTokenRequiredClaimMissing will be returned. -// -// Additionally, if any error occurs while retrieving the claim, e.g., when its -// the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *Validator) verifyAudience(claims Claims, cmp string, required bool) error { - aud, err := claims.GetAudience() - if err != nil { - return err - } - - if len(aud) == 0 { - return errorIfRequired(required, "aud") - } - - // use a var here to keep constant time compare when looping over a number of claims - result := false - - var stringClaims string - for _, a := range aud { - if subtle.ConstantTimeCompare([]byte(a), []byte(cmp)) != 0 { - result = true - } - stringClaims = stringClaims + a - } - - // case where "" is sent in one or many aud claims - if stringClaims == "" { - return errorIfRequired(required, "aud") - } - - return errorIfFalse(result, ErrTokenInvalidAudience) -} - -// verifyIssuer compares the iss claim in claims against cmp. -// -// If iss is not set, it will succeed if the claim is not required, -// otherwise ErrTokenRequiredClaimMissing will be returned. -// -// Additionally, if any error occurs while retrieving the claim, e.g., when its -// the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *Validator) verifyIssuer(claims Claims, cmp string, required bool) error { - iss, err := claims.GetIssuer() - if err != nil { - return err - } - - if iss == "" { - return errorIfRequired(required, "iss") - } - - return errorIfFalse(iss == cmp, ErrTokenInvalidIssuer) -} - -// verifySubject compares the sub claim against cmp. -// -// If sub is not set, it will succeed if the claim is not required, -// otherwise ErrTokenRequiredClaimMissing will be returned. -// -// Additionally, if any error occurs while retrieving the claim, e.g., when its -// the wrong type, an ErrTokenUnverifiable error will be returned. -func (v *Validator) verifySubject(claims Claims, cmp string, required bool) error { - sub, err := claims.GetSubject() - if err != nil { - return err - } - - if sub == "" { - return errorIfRequired(required, "sub") - } - - return errorIfFalse(sub == cmp, ErrTokenInvalidSubject) -} - -// errorIfFalse returns the error specified in err, if the value is true. -// Otherwise, nil is returned. -func errorIfFalse(value bool, err error) error { - if value { - return nil - } else { - return err - } -} - -// errorIfRequired returns an ErrTokenRequiredClaimMissing error if required is -// true. Otherwise, nil is returned. -func errorIfRequired(required bool, claim string) error { - if required { - return newError(fmt.Sprintf("%s claim is required", claim), ErrTokenRequiredClaimMissing) - } else { - return nil - } -} From c7150073acabc77436c8b40f08f84220369efb81 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 18 Apr 2024 16:58:32 +0530 Subject: [PATCH 136/235] make tidy which removed gomock --- go.mod | 1 - go.sum | 3 --- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index 3f2cf98db..3a04ac517 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/gardener/gardener v1.86.0 github.com/go-logr/logr v1.2.4 github.com/golang-jwt/jwt/v5 v5.2.1 - github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 github.com/onsi/ginkgo/v2 v2.13.0 diff --git a/go.sum b/go.sum index f6501f1e9..3197cffe8 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -702,7 +700,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= From bd56d16ddbbc735d2919e8c92980e0181b3e33e6 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 18 Apr 2024 23:02:35 +0530 Subject: [PATCH 137/235] Run `make generate`, set ctrl logger in `it` test suite --- .../10-crd-druid.gardener.cloud_etcds.yaml | 69 ++++++++++++++++++- .../10-crd-druid.gardener.cloud_etcds.yaml | 69 ++++++++++++++++++- test/it/setup/setup.go | 1 + 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index 8bb938ba7..6a66b2b81 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -193,6 +193,12 @@ spec: leadership status of corresponding etcd is checked. type: string type: object + maxBackupsLimitBasedGC: + description: MaxBackupsLimitBasedGC defines the maximum number + of Full snapshots to retain in Limit Based GarbageCollectionPolicy + All full snapshots beyond this limit will be garbage collected. + format: int32 + type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. @@ -1828,8 +1834,69 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: LastError represents the last occurred error. + description: 'LastError represents the last occurred error. Deprecated: + Use LastErrors instead.' type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + description: LastError stores details of the most recent error encountered + for a resource. + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + observedAt: + description: ObservedAt is the time the error was observed. + format: date-time + type: string + required: + - code + - description + - observedAt + type: object + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. + properties: + description: + description: Description describes the last operation. + type: string + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was updated. + format: date-time + type: string + runID: + description: RunID correlates an operation with a reconciliation + run. Every time an etcd resource is reconciled (barring status + reconciliation which is periodic), a unique ID is generated + which can be used to correlate all actions done as part of a + single reconcile run. Capturing this as part of LastOperation + aids in establishing this correlation. This further helps in + also easily filtering reconcile logs as all structured logs + in a reconcile run should have the `runID` referenced. + type: string + state: + description: State is the state of the last operation. + type: string + type: + description: Type is the type of last operation. + type: string + required: + - description + - lastUpdateTime + - runID + - state + - type + type: object members: description: Members represents the members of the etcd cluster items: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 8bb938ba7..6a66b2b81 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -193,6 +193,12 @@ spec: leadership status of corresponding etcd is checked. type: string type: object + maxBackupsLimitBasedGC: + description: MaxBackupsLimitBasedGC defines the maximum number + of Full snapshots to retain in Limit Based GarbageCollectionPolicy + All full snapshots beyond this limit will be garbage collected. + format: int32 + type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. @@ -1828,8 +1834,69 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: LastError represents the last occurred error. + description: 'LastError represents the last occurred error. Deprecated: + Use LastErrors instead.' type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + description: LastError stores details of the most recent error encountered + for a resource. + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + observedAt: + description: ObservedAt is the time the error was observed. + format: date-time + type: string + required: + - code + - description + - observedAt + type: object + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. + properties: + description: + description: Description describes the last operation. + type: string + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was updated. + format: date-time + type: string + runID: + description: RunID correlates an operation with a reconciliation + run. Every time an etcd resource is reconciled (barring status + reconciliation which is periodic), a unique ID is generated + which can be used to correlate all actions done as part of a + single reconcile run. Capturing this as part of LastOperation + aids in establishing this correlation. This further helps in + also easily filtering reconcile logs as all structured logs + in a reconcile run should have the `runID` referenced. + type: string + state: + description: State is the state of the last operation. + type: string + type: + description: Type is the type of last operation. + type: string + required: + - description + - lastUpdateTime + - runID + - state + - type + type: object members: description: Members represents the members of the etcd cluster items: diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index 337067f13..59f877017 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -145,4 +145,5 @@ func (t *itTestEnv) createManager(scheme *k8sruntime.Scheme, clientBuilder *test t.g.Expect(err).ToNot(HaveOccurred()) t.mgr = mgr t.client = mgr.GetClient() + ctrl.SetLogger(logr.Discard()) } From 85d40cb8d423631b2e2528f63ba8c7a021ea8efc Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 18 Apr 2024 23:49:16 +0530 Subject: [PATCH 138/235] Add tool github.com/rakyll/gotest to colorize go test output for better readability --- Makefile | 5 ++--- go.mod | 1 + go.sum | 7 +++++++ hack/test-go.sh | 4 ++-- hack/tools.go | 1 + hack/tools.mk | 7 ++++++- 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b715eeb06..c06e8684f 100644 --- a/Makefile +++ b/Makefile @@ -106,8 +106,7 @@ docker-push: # Run tests .PHONY: test -test: $(GINKGO) -test: $(GINKGO) +test: $(GINKGO) $(GOTEST) @# run ginkgo unit tests. These will be ported to golang native tests over a period of time. @"$(REPO_ROOT)/hack/test.sh" ./api/... \ ./internal/controller/etcdcopybackupstask/... \ @@ -132,7 +131,7 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) @bash $(HACK_DIR)/e2e-test/run-e2e-test.sh $(PROVIDERS) .PHONY: test-integration -test-integration: $(GINKGO) $(SETUP_ENVTEST) +test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTEST) @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test.sh" ./test/integration/... @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test-go.sh" ./test/it/... diff --git a/go.mod b/go.mod index 3a04ac517..3dd59ef81 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 github.com/prometheus/client_golang v1.16.0 + github.com/rakyll/gotest v0.0.6 github.com/spf13/pflag v1.0.5 go.uber.org/mock v0.2.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 3197cffe8..cf52f645f 100644 --- a/go.sum +++ b/go.sum @@ -134,6 +134,7 @@ github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fluent/fluent-operator/v2 v2.2.0 h1:97CiP6WKOHRM7zY/zCynX187Rg+T8hgx2JzD2iuJof8= @@ -358,6 +359,7 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= @@ -365,6 +367,8 @@ github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HN github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -432,6 +436,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rakyll/gotest v0.0.6 h1:hBTqkO3jiuwYW/M9gL4bu0oTYcm8J6knQAAPUsJsz1I= +github.com/rakyll/gotest v0.0.6/go.mod h1:SkoesdNCWmiD4R2dljIUcfSnNdVZ12y8qK4ojDkc2Sc= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -621,6 +627,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/hack/test-go.sh b/hack/test-go.sh index 76f9a54e3..8b7daad97 100755 --- a/hack/test-go.sh +++ b/hack/test-go.sh @@ -36,9 +36,9 @@ if ${TEST_COV:-false}; then output_dir=test/output coverprofile_file=coverprofile.out mkdir -p test/output - go test -v -coverprofile=cover.out $@ + gotest -v -coverprofile=cover.out $@ go tool cover -func=cover.out exit 0 fi -go test -v $@ +gotest -v $@ diff --git a/hack/tools.go b/hack/tools.go index 140823a11..573812cf9 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -13,5 +13,6 @@ import ( _ "github.com/gardener/gardener/hack" _ "github.com/gardener/gardener/hack/.ci" _ "github.com/onsi/ginkgo/v2/ginkgo" + _ "github.com/rakyll/gotest" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" ) diff --git a/hack/tools.mk b/hack/tools.mk index 438c2c7c2..a1730eb75 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -5,10 +5,12 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/bin SKAFFOLD := $(TOOLS_BIN_DIR)/skaffold KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize +GOTEST := $(TOOLS_BIN_DIR)/gotest # default tool versions SKAFFOLD_VERSION ?= v1.38.0 KUSTOMIZE_VERSION ?= v4.5.7 +GOTEST_VERSION ?= v0.0.6 export TOOLS_BIN_DIR := $(TOOLS_BIN_DIR) export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) @@ -18,4 +20,7 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) ######################################### $(KUSTOMIZE): - @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION} \ No newline at end of file + @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION} + +$(GOTEST): $(call tool_version_file,$(GOTEST),$(GOTEST_VERSION)) + go build -o $(GOTEST) github.com/rakyll/gotest From 21b226c2ff9432e6ad93932e025052248fd4d477 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sat, 20 Apr 2024 11:41:52 +0530 Subject: [PATCH 139/235] Fix webhook config in manager; Add generic-garbage-collector to sentinel webhook exempt SAs --- charts/druid/values.yaml | 3 ++- internal/controller/manager.go | 15 +++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index c82a3f12c..76a31ac5c 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -22,4 +22,5 @@ server: sentinelWebhook: enabled: true - exemptServiceAccounts: {} + exemptServiceAccounts: + - system:serviceaccount:kube-system:generic-garbage-collector diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 3f2172266..c6fc77e70 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -14,7 +14,6 @@ import ( "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" "github.com/gardener/etcd-druid/internal/webhook/sentinel" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" @@ -23,6 +22,8 @@ import ( eventsv1beta1 "k8s.io/api/events/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) var ( @@ -69,10 +70,16 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { }, Scheme: kubernetes.Scheme, Metrics: metricsserver.Options{ - BindAddress: config.MetricsAddr, + BindAddress: config.Server.Metrics.BindAddress, }, - LeaderElection: config.EnableLeaderElection, - LeaderElectionID: config.LeaderElectionID, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: config.Server.Webhook.BindAddress, + Port: config.Server.Webhook.Port, + CertDir: config.Server.Webhook.TLS.ServerCertDir, + }), + LeaderElection: config.EnableLeaderElection, + LeaderElectionID: config.LeaderElectionID, + LeaderElectionResourceLock: config.LeaderElectionResourceLock, }) } From 071bb534213de4b7eae0b45b7a572c1107d4ac05 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sat, 20 Apr 2024 11:44:07 +0530 Subject: [PATCH 140/235] Don't swallow infra job apt command output --- .../infrastructure/overlays/aws/common/files/common.sh | 6 +++--- .../infrastructure/overlays/azure/common/files/common.sh | 4 ++-- .../infrastructure/overlays/gcp/common/files/common.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh b/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh index cf2f45b74..428e5beeb 100644 --- a/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh +++ b/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh @@ -14,9 +14,9 @@ function setup_aws() { return fi echo "Installing awscli..." - apt update > /dev/null - apt install -y curl > /dev/null - apt install -y unzip > /dev/null + apt update + apt install -y curl + apt install -y unzip cd $HOME curl -Lo "awscliv2.zip" "https://awscli.amazonaws.com/awscli-exe-$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m).zip" unzip awscliv2.zip > /dev/null diff --git a/hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh b/hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh index bd9793116..c5dbd1a54 100644 --- a/hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh +++ b/hack/e2e-test/infrastructure/overlays/azure/common/files/common.sh @@ -9,8 +9,8 @@ function setup_azcli() { return fi echo "Installing azure-cli..." - apt update > /dev/null - apt install -y curl > /dev/null + apt update + apt install -y curl curl -sL https://aka.ms/InstallAzureCLIDeb | bash echo "Successfully installed azure-cli." } diff --git a/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh b/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh index b2a6c7557..f733cdc0b 100644 --- a/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh +++ b/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh @@ -10,9 +10,9 @@ function setup_gcloud() { fi echo "Installing gcloud..." cd $HOME - apt update > /dev/null - apt install -y curl > /dev/null - apt install -y python3 > /dev/null + apt update + apt install -y curl + apt install -y python3 curl -Lo "google-cloud-sdk.tar.gz" https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m | sed 's/aarch64/arm/').tar.gz tar -xzf google-cloud-sdk.tar.gz ./google-cloud-sdk/install.sh -q From cb7afa10cf90903b229115144631e742a7929763 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sat, 20 Apr 2024 11:44:28 +0530 Subject: [PATCH 141/235] Fix e2e tests --- test/e2e/etcd_backup_test.go | 22 +++++++++++----------- test/e2e/etcd_compaction_test.go | 22 +++++++++++----------- test/e2e/etcd_multi_node_test.go | 10 +++++----- test/e2e/suite_test.go | 3 +++ test/e2e/utils.go | 32 ++++++++++++++++---------------- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index 17f071c09..b4364e3b0 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -53,6 +53,17 @@ var _ = Describe("Etcd Backup", func() { etcdName = fmt.Sprintf("etcd-%s", provider.Name) storageContainer = getEnvAndExpectNoError(envStorageContainer) + + By("Purge snapstore") + snapstoreProvider := provider.Storage.Provider + if snapstoreProvider == utils.Local { + purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) + defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) + } else { + store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) + Expect(err).ShouldNot(HaveOccurred()) + Expect(purgeSnapstore(store)).To(Succeed()) + } }) AfterEach(func() { @@ -66,17 +77,6 @@ var _ = Describe("Etcd Backup", func() { By("Purge etcd") purgeEtcd(ctx, cl, providers) - - By("Purge snapstore") - snapstoreProvider := provider.Storage.Provider - if snapstoreProvider == utils.Local { - purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) - defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) - } else { - store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) - Expect(err).ShouldNot(HaveOccurred()) - Expect(purgeSnapstore(store)).To(Succeed()) - } }) It("Should create, test backup and delete etcd with backup", func() { diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index 58e0cd0a9..f19735719 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -47,6 +47,17 @@ var _ = Describe("Etcd Compaction", func() { etcdName = fmt.Sprintf("etcd-%s", provider.Name) storageContainer = getEnvAndExpectNoError(envStorageContainer) + + By("Purge snapstore") + snapstoreProvider := provider.Storage.Provider + if snapstoreProvider == utils.Local { + purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) + defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) + } else { + store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) + Expect(err).ShouldNot(HaveOccurred()) + Expect(purgeSnapstore(store)).To(Succeed()) + } }) AfterEach(func() { @@ -60,17 +71,6 @@ var _ = Describe("Etcd Compaction", func() { By("Purge etcd") purgeEtcd(ctx, cl, providers) - - By("Purge snapstore") - snapstoreProvider := provider.Storage.Provider - if snapstoreProvider == utils.Local { - purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) - defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) - } else { - store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix) - Expect(err).ShouldNot(HaveOccurred()) - Expect(purgeSnapstore(store)).To(Succeed()) - } }) It("should test compaction on backup", func() { diff --git a/test/e2e/etcd_multi_node_test.go b/test/e2e/etcd_multi_node_test.go index 729500c84..df7701e1b 100644 --- a/test/e2e/etcd_multi_node_test.go +++ b/test/e2e/etcd_multi_node_test.go @@ -50,11 +50,6 @@ var _ = Describe("Etcd", func() { etcdName = fmt.Sprintf("etcd-%s", provider.Name) storageContainer = getEnvAndExpectNoError(envStorageContainer) - }) - - AfterEach(func() { - By("Purge etcd") - purgeEtcd(parentCtx, cl, providers) By("Purge snapstore") snapstoreProvider := provider.Storage.Provider @@ -68,6 +63,11 @@ var _ = Describe("Etcd", func() { } }) + AfterEach(func() { + By("Purge etcd") + purgeEtcd(parentCtx, cl, providers) + }) + Context("when multi-node is configured", func() { It("should perform etcd operations", func() { ctx, cancelFunc := context.WithTimeout(parentCtx, 15*time.Minute) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index f5be8a0f7..d4bf5898b 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -13,6 +13,7 @@ import ( "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/gardener/pkg/utils/test/matchers" + "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -21,6 +22,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) @@ -73,6 +75,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) logger.V(1).Info("setting up k8s client", "KUBECONFIG", kubeconfigPath) + log.SetLogger(logr.Discard()) cl, err = getKubernetesClient(kubeconfigPath) Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 4a4878926..7bdcf3b38 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -576,7 +576,7 @@ func getRemoteCommandExecutor(kubeconfigPath, namespace, podName, containerName, // executeRemoteCommand executes a remote shell command on the given pod and container // and returns the stdout and stderr logs -func executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, command string) (string, string, error) { +func executeRemoteCommand(ctx context.Context, kubeconfigPath, namespace, podName, containerName, command string) (string, string, error) { exec, err := getRemoteCommandExecutor(kubeconfigPath, namespace, podName, containerName, command) if err != nil { return "", "", err @@ -584,7 +584,7 @@ func executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, com buf := &bytes.Buffer{} errBuf := &bytes.Buffer{} - err = exec.Stream(remotecommand.StreamOptions{ + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ Stdout: buf, Stderr: errBuf, }) @@ -663,7 +663,7 @@ func getPurgeLocalSnapstoreJob(storeContainer string) *batchv1.Job { ) } -func populateEtcd(logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, valuePrefix string, startKeyNo, endKeyNo int, delay time.Duration) error { +func populateEtcd(ctx context.Context, logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, valuePrefix string, startKeyNo, endKeyNo int, delay time.Duration) error { var ( cmd string stdout string @@ -679,14 +679,14 @@ func populateEtcd(logger logr.Logger, kubeconfigPath, namespace, etcdName, podNa "rm -rf etcd-$ETCD_VERSION-linux-amd64;'" + " > test.sh && sh test.sh" - stdout, stderr, err = executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, install) + stdout, stderr, err = executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, install) if err != nil { logger.Error(err, fmt.Sprintf("Failed to inatall etcdctl. err: %s, stderr: %s, stdout:%s", err, stderr, stdout)) } for i := startKeyNo; i <= endKeyNo; { cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://%s-client.shoot.svc:%d --cacert /var/etcd/ssl/ca/ca.crt --cert=/var/etcd/ssl/client/tls.crt --key=/var/etcd/ssl/client/tls.key put %s-%d %s-%d", etcdName, etcdClientPort, keyPrefix, i, valuePrefix, i) - stdout, stderr, err = executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err = executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stderr != "" || stdout != "OK" { logger.Error(err, fmt.Sprintf("failed to put (%s-%d, %s-%d): stdout: %s; stderr: %s. Retrying", keyPrefix, i, valuePrefix, i, stdout, stderr)) retries++ @@ -703,7 +703,7 @@ func populateEtcd(logger logr.Logger, kubeconfigPath, namespace, etcdName, podNa return nil } -func getEtcdKey(kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, suffix int) (string, string, error) { +func getEtcdKey(ctx context.Context, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, suffix int) (string, string, error) { var ( cmd string stdout string @@ -712,7 +712,7 @@ func getEtcdKey(kubeconfigPath, namespace, etcdName, podName, containerName, key ) cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://%s-client.shoot.svc:%d --cacert /var/etcd/ssl/ca/ca.crt --cert=/var/etcd/ssl/client/tls.crt --key=/var/etcd/ssl/client/tls.key get %s-%d", etcdName, etcdClientPort, keyPrefix, suffix) - stdout, stderr, err = executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err = executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stderr != "" { return "", "", fmt.Errorf("failed to get %s-%d: stdout: %s; stderr: %s; err: %v", keyPrefix, suffix, stdout, stderr, err) } @@ -724,7 +724,7 @@ func getEtcdKey(kubeconfigPath, namespace, etcdName, podName, containerName, key return strings.TrimSpace(splits[0]), strings.TrimSpace(splits[1]), nil } -func getEtcdKeys(logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, start, end int) (map[string]string, error) { +func getEtcdKeys(ctx context.Context, logger logr.Logger, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix string, start, end int) (map[string]string, error) { var ( key string val string @@ -733,7 +733,7 @@ func getEtcdKeys(logger logr.Logger, kubeconfigPath, namespace, etcdName, podNam err error ) for i := start; i <= end; { - key, val, err = getEtcdKey(kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, i) + key, val, err = getEtcdKey(ctx, kubeconfigPath, namespace, etcdName, podName, containerName, keyPrefix, i) if err != nil { logger.Info("failed to get key. Retrying...", "key", fmt.Sprintf("%s-%d", keyPrefix, i)) retries++ @@ -751,7 +751,7 @@ func getEtcdKeys(logger logr.Logger, kubeconfigPath, namespace, etcdName, podNam return keyValueMap, nil } -func triggerOnDemandSnapshot(kubeconfigPath, namespace, etcdName, podName, containerName string, port int, snapshotKind string) (*brtypes.Snapshot, error) { +func triggerOnDemandSnapshot(ctx context.Context, kubeconfigPath, namespace, etcdName, podName, containerName string, port int, snapshotKind string) (*brtypes.Snapshot, error) { var ( snapshot *brtypes.Snapshot snapKind string @@ -766,7 +766,7 @@ func triggerOnDemandSnapshot(kubeconfigPath, namespace, etcdName, podName, conta return nil, fmt.Errorf("invalid snapshotKind %s", snapshotKind) } cmd := fmt.Sprintf("curl https://%s-client.shoot.svc:%d/snapshot/%s -k -s", etcdName, port, snapKind) - stdout, stderr, err := executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err := executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stdout == "" { return nil, fmt.Errorf("failed to trigger on-demand %s snapshot for %s: stdout: %s; stderr: %s; err: %v", snapKind, podName, stdout, stderr, err) } @@ -777,10 +777,10 @@ func triggerOnDemandSnapshot(kubeconfigPath, namespace, etcdName, podName, conta return snapshot, nil } -func getLatestSnapshots(kubeconfigPath, namespace, etcdName, podName, containerName string, port int) (*LatestSnapshots, error) { +func getLatestSnapshots(ctx context.Context, kubeconfigPath, namespace, etcdName, podName, containerName string, port int) (*LatestSnapshots, error) { var latestSnapshots *LatestSnapshots cmd := fmt.Sprintf("curl https://%s-client.shoot.svc:%d/snapshot/latest -k -s", etcdName, port) - stdout, stderr, err := executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err := executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stdout == "" { return nil, fmt.Errorf("failed to fetch latest snapshots taken for %s: stdout: %s; stderr: %s; err: %v", podName, stdout, stderr, err) } @@ -796,9 +796,9 @@ func getLatestSnapshots(kubeconfigPath, namespace, etcdName, podName, containerN return latestSnapshots, nil } -func deleteDir(kubeconfigPath, namespace, podName, containerName string, dirPath string) error { +func deleteDir(ctx context.Context, kubeconfigPath, namespace, podName, containerName string, dirPath string) error { cmd := fmt.Sprintf("rm -rf %s", dirPath) - stdout, stderr, err := executeRemoteCommand(kubeconfigPath, namespace, podName, containerName, cmd) + stdout, stderr, err := executeRemoteCommand(ctx, kubeconfigPath, namespace, podName, containerName, cmd) if err != nil || stdout != "" { return fmt.Errorf("failed to delete directory %s for %s: stdout: %s; stderr: %s; err: %v", dirPath, podName, stdout, stderr, err) } @@ -875,7 +875,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon "failed=0 ; threshold=2 ; " + "while [ $failed -lt $threshold ] ; do " + "$(curl --cacert /var/etcd/ssl/ca/ca.crt --cert /var/etcd/ssl/client/tls.crt --key /var/etcd/ssl/client/tls.key https://" + etcdSvc + ":2379/health -s -f -o /dev/null ); " + - "if [ $? -gt 0 ] ; then let failed++; echo \"etcd is unhealthy and retrying\"; sleep 1; continue; fi ; " + + "if [ $? -gt 0 ] ; then let failed++; echo \"etcd is unhealthy and retrying\"; sleep 2; continue; fi ; " + "echo \"etcd is healthy\"; touch /tmp/healthy; let failed=0; " + "sleep 2; done; echo \"etcd is unhealthy\"; exit 1;" + "' > test.sh && sh test.sh", From 9b38eb9ef2695191e57dcfc5988d584e3acf6788 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Apr 2024 10:52:42 +0530 Subject: [PATCH 142/235] fixed golangci lint errors, now use gotestfmt, refactored etcd it tests --- .golangci.yaml | 18 ++- Makefile | 2 +- hack/test-go.sh | 11 +- hack/tools.mk | 8 +- internal/controller/compaction/reconciler.go | 5 +- internal/controller/secret/reconciler_test.go | 7 +- .../check_data_volumes_ready_test.go | 19 ++- internal/operator/statefulset/builder.go | 3 +- test/it/controller/etcd/reconciler_test.go | 103 ++++++++------ test/it/setup/setup.go | 134 ++++++++++-------- 10 files changed, 178 insertions(+), 132 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 8715615e5..df3aeef25 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,9 +1,6 @@ run: concurrency: 4 - deadline: 10m - - skip-files: - - "zz_generated\\..*\\.go$" + timeout: 10m linters: disable: @@ -32,3 +29,16 @@ issues: - linters: - staticcheck text: "SA1019:" # Excludes messages where deprecated variables are used + + exclude-files: + - "zz_generated\\..*\\.go$" + +linters-settings: + revive: + rules: + - name: duplicated-imports + - name: unused-parameter + - name: unreachable-code + - name: context-as-argument + - name: early-return + - name: exported diff --git a/Makefile b/Makefile index c06e8684f..b49742125 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ docker-push: # Run tests .PHONY: test -test: $(GINKGO) $(GOTEST) +test: $(GINKGO) $(GOTESTFMT) @# run ginkgo unit tests. These will be ported to golang native tests over a period of time. @"$(REPO_ROOT)/hack/test.sh" ./api/... \ ./internal/controller/etcdcopybackupstask/... \ diff --git a/hack/test-go.sh b/hack/test-go.sh index 8b7daad97..e85ee9b08 100755 --- a/hack/test-go.sh +++ b/hack/test-go.sh @@ -12,8 +12,8 @@ ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.22"} if ${SETUP_ENVTEST:-false}; then echo "> Installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest" - if ! command -v setup-envtest &> /dev/null ; then - >&2 echo "setup-envtest not available" + if ! command -v setup-envtest &>/dev/null; then + echo >&2 "setup-envtest not available" exit 1 fi @@ -33,12 +33,9 @@ fi echo "> Go tests" if ${TEST_COV:-false}; then - output_dir=test/output - coverprofile_file=coverprofile.out mkdir -p test/output - gotest -v -coverprofile=cover.out $@ - go tool cover -func=cover.out + go test -json -cover "$@" | gotestfmt -hide empty-packages exit 0 fi -gotest -v $@ +gotest -v "$@" diff --git a/hack/tools.mk b/hack/tools.mk index a1730eb75..98f94c09a 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -5,12 +5,12 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/bin SKAFFOLD := $(TOOLS_BIN_DIR)/skaffold KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize -GOTEST := $(TOOLS_BIN_DIR)/gotest +GOTESTFMT := $(TOOLS_BIN_DIR)/gotestfmt # default tool versions SKAFFOLD_VERSION ?= v1.38.0 KUSTOMIZE_VERSION ?= v4.5.7 -GOTEST_VERSION ?= v0.0.6 +GOTESTFMT_VERSION ?= v2.5.0 export TOOLS_BIN_DIR := $(TOOLS_BIN_DIR) export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) @@ -22,5 +22,5 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) $(KUSTOMIZE): @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION} -$(GOTEST): $(call tool_version_file,$(GOTEST),$(GOTEST_VERSION)) - go build -o $(GOTEST) github.com/rakyll/gotest +$(GOTESTFMT): + GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@$(GOTESTFMT_VERSION) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 6470d9e50..35a34d6fe 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -103,14 +103,13 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd // Update metrics for currently running compaction job, if any job := &batchv1.Job{} if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { - if errors.IsNotFound(err) { - logger.Info("No compaction job currently running", "namespace", etcd.Namespace) - } else { + if !errors.IsNotFound(err) { // Error reading the object - requeue the request. return ctrl.Result{ RequeueAfter: 10 * time.Second, }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", etcd.GetCompactionJobName(), etcd.Namespace, err) } + logger.Info("No compaction job currently running", "namespace", etcd.Namespace) } if job != nil && job.Name != "" { diff --git a/internal/controller/secret/reconciler_test.go b/internal/controller/secret/reconciler_test.go index c2d45be7d..260e1996b 100644 --- a/internal/controller/secret/reconciler_test.go +++ b/internal/controller/secret/reconciler_test.go @@ -14,7 +14,6 @@ import ( "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -57,7 +56,7 @@ var _ = Describe("SecretController", func() { It("should return true if secret is referred to in an Etcd object's Spec.Etcd.ClientUrlTLS section", func() { etcdList.Items[0].Spec.Etcd.ClientUrlTLS = &druidv1alpha1.TLSConfig{ - ServerTLSSecretRef: v1.SecretReference{ + ServerTLSSecretRef: corev1.SecretReference{ Name: testSecretName, Namespace: testNamespace, }, @@ -68,7 +67,7 @@ var _ = Describe("SecretController", func() { It("should return true if secret is referred to in an Etcd object's Spec.Etcd.PeerUrlTLS section", func() { etcdList.Items[1].Spec.Etcd.PeerUrlTLS = &druidv1alpha1.TLSConfig{ - ServerTLSSecretRef: v1.SecretReference{ + ServerTLSSecretRef: corev1.SecretReference{ Name: testSecretName, Namespace: testNamespace, }, @@ -79,7 +78,7 @@ var _ = Describe("SecretController", func() { It("should return true if secret is referred to in an Etcd object's Spec.Backup.Store section", func() { etcdList.Items[2].Spec.Backup.Store = &druidv1alpha1.StoreSpec{ - SecretRef: &v1.SecretReference{ + SecretRef: &corev1.SecretReference{ Name: testSecretName, Namespace: testNamespace, }, diff --git a/internal/health/condition/check_data_volumes_ready_test.go b/internal/health/condition/check_data_volumes_ready_test.go index 0e4b134e5..d02f4270d 100644 --- a/internal/health/condition/check_data_volumes_ready_test.go +++ b/internal/health/condition/check_data_volumes_ready_test.go @@ -18,7 +18,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,18 +30,18 @@ var _ = Describe("DataVolumesReadyCheck", func() { cl *mockclient.MockClient notFoundErr = apierrors.StatusError{ - ErrStatus: v1.Status{ - Reason: v1.StatusReasonNotFound, + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, }, } internalErr = apierrors.StatusError{ - ErrStatus: v1.Status{ - Reason: v1.StatusReasonInternalError, + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonInternalError, }, } etcd = druidv1alpha1.Etcd{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", }, @@ -51,7 +50,7 @@ var _ = Describe("DataVolumesReadyCheck", func() { }, } sts = &appsv1.StatefulSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, @@ -60,7 +59,7 @@ var _ = Describe("DataVolumesReadyCheck", func() { Replicas: pointer.Int32(1), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ { - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "test", }, }, @@ -72,7 +71,7 @@ var _ = Describe("DataVolumesReadyCheck", func() { }, } pvc = &corev1.PersistentVolumeClaim{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "test-test-0", Namespace: "default", }, @@ -81,7 +80,7 @@ var _ = Describe("DataVolumesReadyCheck", func() { }, } event = &corev1.Event{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: "test-event", Namespace: "default", }, diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 68b40fb80..c93479b73 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -13,7 +13,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" - druidutils "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -80,7 +79,7 @@ func newStsBuilder(client client.Client, useEtcdWrapper bool, imageVector imagevector.ImageVector, sts *appsv1.StatefulSet) (*stsBuilder, error) { - etcdImage, etcdBackupRestoreImage, initContainerImage, err := druidutils.GetEtcdImages(etcd, imageVector, useEtcdWrapper) + etcdImage, etcdBackupRestoreImage, initContainerImage, err := utils.GetEtcdImages(etcd, imageVector, useEtcdWrapper) if err != nil { return nil, err } diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 6e04575db..2409f3e70 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "encoding/hex" "fmt" + "os" "testing" "time" @@ -38,31 +39,27 @@ import ( const testNamespacePrefix = "etcd-reconciler-test-" -// TestSuiteWithDefaultIntegrationTestEnvironment runs the etcd reconciler tests with the default integration test environment. -// No auto-reconcile is set, to reconcile one needs to set the GardenerOperation annotation. -// All tests defined in this suite share the same integration test environment. -func TestSuiteWithDefaultIntegrationTestEnvironment(t *testing.T) { - itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnv(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}) - defer itTestEnvCloser(t) - reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) +var ( + sharedITTestEnv setup.IntegrationTestEnv +) - tests := []struct { - name string - testFn func(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) - }{ - {"test spec reconciliation without auto-reconcile", testEtcdReconcileSpec}, - {"test deletion of all etcd resources when etcd marked for deletion", testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion}, - {"test etcd status reconciliation", testEtcdStatus}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - test.testFn(t, reconcilerTestEnv) - }) +func TestMain(m *testing.M) { + var ( + itTestEnvCloser setup.IntegrationTestEnvCloser + err error + ) + sharedITTestEnv, itTestEnvCloser, err = setup.NewIntegrationTestEnv("etcd-reconciler", []string{assets.GetEtcdCrdPath()}) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "failed to create integration test environment: %v\n", err) + os.Exit(1) } + defer itTestEnvCloser() + os.Exit(m.Run()) } // ------------------------------ reconcile spec tests ------------------------------ -func testEtcdReconcileSpec(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { +func TestEtcdReconcileSpecWithNoAutoReconcile(t *testing.T) { + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, sharedITTestEnv, false, testutils.NewTestClientBuilder()) tests := []struct { name string fn func(t *testing.T, testNamespace string, reconcilerTestEnv ReconcilerTestEnv) @@ -72,11 +69,12 @@ func testEtcdReconcileSpec(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { {"should not reconcile spec when reconciliation is suspended", testWhenReconciliationIsSuspended}, {"should not reconcile when no reconcile operation annotation is set", testWhenNoReconcileOperationAnnotationIsSet}, } + g := NewWithT(t) for _, test := range tests { t.Run(test.name, func(t *testing.T) { testNs := createTestNamespaceName(t) t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) - reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs) + g.Expect(reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs)).To(Succeed()) test.fn(t, testNs, reconcilerTestEnv) }) } @@ -202,14 +200,31 @@ func testWhenNoReconcileOperationAnnotationIsSet(t *testing.T, testNs string, re } // ------------------------------ reconcile deletion tests ------------------------------ -func testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { - // --------------------------- create test namespace --------------------------- - testNs := createTestNamespaceName(t) - reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs) - t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) - // ---------------------------- create etcd instance -------------------------- +func TestEtcdDeletion(t *testing.T) { + tests := []struct { + name string + fn func(t *testing.T, testNamespace string) + }{ + {"test deletion of all etcd resources when etcd marked for deletion", testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion}, + {"test partial deletion failure of etcd resources when etcd marked for deletion", testPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNs := createTestNamespaceName(t) + test.fn(t, testNs) + }) + } +} + +func testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T, testNs string) { g := NewWithT(t) + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, sharedITTestEnv, false, testutils.NewTestClientBuilder()) + // ---------------------------- create test namespace --------------------------- + t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) + g.Expect(sharedITTestEnv.CreateTestNamespace(testNs)).To(Succeed()) + // ---------------------------- create etcd instance -------------------------- etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). WithClientTLS(). WithPeerTLS(). @@ -230,9 +245,8 @@ func testDeletionOfAllEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T, recon assertETCDFinalizer(t, cl, client.ObjectKeyFromObject(etcdInstance), false, 2*time.Minute, 2*time.Second) } -func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T) { +func testPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testing.T, testNs string) { // ********************************** setup ********************************** - testNs := createTestNamespaceName(t) etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). WithClientTLS(). WithPeerTLS(). @@ -250,16 +264,20 @@ func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: etcdInstance.Name, }, testutils.TestAPIInternalErr) - // create the integration test environment with the test client builder - itTestEnv, itTestEnvCloser := setup.NewIntegrationTestEnvWithClientBuilder(t, "etcd-reconciler", []string{assets.GetEtcdCrdPath()}, testClientBuilder) - defer itTestEnvCloser(t) - reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false) - // create test namespace - itTestEnv.CreateTestNamespace(testNs) + g := NewWithT(t) + + // A different IT test environment is required due to a different clientBuilder which is used to create the manager. + itTestEnv, itTestEnvCloser, err := setup.NewIntegrationTestEnv("etcd-reconciler", []string{assets.GetEtcdCrdPath()}) + g.Expect(err).ToNot(HaveOccurred()) + defer itTestEnvCloser() + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, itTestEnv, false, testClientBuilder) + + // ---------------------------- create test namespace --------------------------- t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) + g.Expect(itTestEnv.CreateTestNamespace(testNs)).To(Succeed()) - g := NewWithT(t) + // ---------------------------- create etcd instance -------------------------- ctx := context.Background() cl := itTestEnv.GetClient() @@ -285,6 +303,7 @@ func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi operator.RoleKind, operator.RoleBindingKind, operator.StatefulSetKind}, 2*time.Minute, 2*time.Second) + assertSelectedComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, []operator.Kind{operator.ClientServiceKind, operator.SnapshotLeaseKind}, 2*time.Minute, 2*time.Second) // assert that the last operation and last errors are updated correctly. expectedLastOperation := &druidv1alpha1.LastOperation{ @@ -297,7 +316,9 @@ func TestPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi } // ------------------------------ reconcile status tests ------------------------------ -func testEtcdStatus(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { + +func TestEtcdStatusReconciliation(t *testing.T) { + reconcilerTestEnv := initializeEtcdReconcilerTestEnv(t, sharedITTestEnv, false, testutils.NewTestClientBuilder()) tests := []struct { name string fn func(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) @@ -315,11 +336,12 @@ func testEtcdStatus(t *testing.T, reconcilerTestEnv ReconcilerTestEnv) { } ctx := context.Background() + g := NewWithT(t) for _, test := range tests { t.Run(test.name, func(t *testing.T) { // --------------------------- create test namespace --------------------------- testNs := createTestNamespaceName(t) - reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs) + g.Expect(reconcilerTestEnv.itTestEnv.CreateTestNamespace(testNs)).To(Succeed()) t.Logf("successfully create namespace: %s to run test => '%s'", testNs, t.Name()) // ---------------------------- create etcd instance -------------------------- etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). @@ -458,12 +480,13 @@ func generateRandomAlphanumericString(t *testing.T, length int) string { return hex.EncodeToString(b) } -func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTestEnv, autoReconcile bool) ReconcilerTestEnv { +func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTestEnv, autoReconcile bool, clientBuilder *testutils.TestClientBuilder) ReconcilerTestEnv { g := NewWithT(t) var ( reconciler *etcd.Reconciler err error ) + g.Expect(itTestEnv.CreateManager(clientBuilder)).To(Succeed()) itTestEnv.RegisterReconciler(func(mgr manager.Manager) { reconciler, err = etcd.NewReconcilerWithImageVector(mgr, &etcd.Config{ @@ -482,7 +505,7 @@ func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTe g.Expect(err).ToNot(HaveOccurred()) g.Expect(reconciler.RegisterWithManager(mgr)).To(Succeed()) }) - itTestEnv.StartManager() + g.Expect(itTestEnv.StartManager()).To(Succeed()) t.Log("successfully registered etcd reconciler with manager and started manager") return ReconcilerTestEnv{ itTestEnv: itTestEnv, diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index 59f877017..cbebb9a40 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -6,7 +6,6 @@ package setup import ( "context" - "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" testutils "github.com/gardener/etcd-druid/test/utils" @@ -15,7 +14,6 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/go-logr/logr" - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -29,11 +27,16 @@ import ( type AddToManagerFn func(mgr manager.Manager) -type IntegrationTestEnv interface { +type ManagerRegisterer interface { RegisterReconciler(addToMgrFn AddToManagerFn) - StartManager() + CreateManager(clientBuilder *testutils.TestClientBuilder) error + StartManager() error +} + +type IntegrationTestEnv interface { + ManagerRegisterer GetClient() client.Client - CreateTestNamespace(name string) + CreateTestNamespace(name string) error GetLogger() logr.Logger GetContext() context.Context } @@ -41,59 +44,95 @@ type IntegrationTestEnv interface { type itTestEnv struct { ctx context.Context cancelFn context.CancelFunc - g *WithT mgr manager.Manager client client.Client config *rest.Config + scheme *k8sruntime.Scheme testEnv *envtest.Environment logger logr.Logger } -type IntegrationTestEnvCloser func(t *testing.T) +type IntegrationTestEnvCloser func() -func NewIntegrationTestEnvWithClientBuilder(t *testing.T, loggerName string, crdDirectoryPaths []string, clientBuilder *testutils.TestClientBuilder) (IntegrationTestEnv, IntegrationTestEnvCloser) { +func NewIntegrationTestEnv(loggerName string, crdDirectoryPaths []string) (IntegrationTestEnv, IntegrationTestEnvCloser, error) { ctx, cancelFunc := context.WithCancel(context.Background()) + logger := ctrl.Log.WithName(loggerName) itEnv := &itTestEnv{ ctx: ctx, cancelFn: cancelFunc, - g: NewWithT(t), - logger: ctrl.Log.WithName(loggerName), + logger: logger, } - druidScheme := itEnv.prepareScheme() - itEnv.createTestEnvironment(druidScheme, crdDirectoryPaths) - itEnv.createManager(druidScheme, clientBuilder) - return itEnv, func(t *testing.T) { - itEnv.cancelFn() - itEnv.g.Expect(itEnv.testEnv.Stop()).To(Succeed()) - t.Log("stopped test environment") + if err := itEnv.prepareScheme(); err != nil { + return nil, nil, err } -} - -func NewIntegrationTestEnv(t *testing.T, loggerName string, crdDirectoryPaths []string) (IntegrationTestEnv, IntegrationTestEnvCloser) { - return NewIntegrationTestEnvWithClientBuilder(t, loggerName, crdDirectoryPaths, testutils.NewTestClientBuilder()) + if err := itEnv.createTestEnvironment(crdDirectoryPaths); err != nil { + return nil, nil, err + } + return itEnv, func() { + itEnv.cancelFn() + err := itEnv.testEnv.Stop() + if err != nil { + logger.Error(err, "failed to stop test environment") + } + logger.Info("stopped test environment") + }, nil } func (t *itTestEnv) RegisterReconciler(addToMgrFn AddToManagerFn) { addToMgrFn(t.mgr) } -func (t *itTestEnv) StartManager() { +func (t *itTestEnv) CreateManager(clientBuilder *testutils.TestClientBuilder) error { + mgr, err := manager.New(t.config, manager.Options{ + Scheme: t.scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { + options.Cache.DisableFor = []client.Object{ + &corev1.Event{}, + &eventsv1beta1.Event{}, + &eventsv1.Event{}, + } + cl, err := client.New(config, options) + if err != nil { + return nil, err + } + testCl := clientBuilder.WithClient(cl).Build() + return testCl, nil + }, + }) + if err != nil { + return err + } + t.mgr = mgr + t.client = mgr.GetClient() + ctrl.SetLogger(logr.Discard()) + return nil +} + +func (t *itTestEnv) StartManager() (err error) { go func() { - t.g.Expect(t.mgr.Start(t.ctx)).To(Succeed()) + err = t.mgr.Start(t.ctx) + if err != nil { + t.logger.Error(err, "failed to start manager") + return + } }() + return err } func (t *itTestEnv) GetClient() client.Client { return t.client } -func (t *itTestEnv) CreateTestNamespace(name string) { +func (t *itTestEnv) CreateTestNamespace(name string) error { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, } - t.g.Expect(t.client.Create(t.ctx, ns)).To(Succeed()) + return t.client.Create(t.ctx, ns) } func (t *itTestEnv) GetLogger() logr.Logger { @@ -104,46 +143,27 @@ func (t *itTestEnv) GetContext() context.Context { return t.ctx } -func (t *itTestEnv) prepareScheme() *k8sruntime.Scheme { - t.g.Expect(druidv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) - return scheme.Scheme +func (t *itTestEnv) prepareScheme() error { + if err := druidv1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } else { + t.scheme = scheme.Scheme + } + return nil } -func (t *itTestEnv) createTestEnvironment(scheme *k8sruntime.Scheme, crdDirectoryPaths []string) { +func (t *itTestEnv) createTestEnvironment(crdDirectoryPaths []string) error { testEnv := &envtest.Environment{ - Scheme: scheme, + Scheme: t.scheme, ErrorIfCRDPathMissing: true, CRDDirectoryPaths: crdDirectoryPaths, } cfg, err := testEnv.Start() - t.g.Expect(err).ToNot(HaveOccurred()) + if err != nil { + return err + } t.config = cfg t.testEnv = testEnv -} - -func (t *itTestEnv) createManager(scheme *k8sruntime.Scheme, clientBuilder *testutils.TestClientBuilder) { - mgr, err := manager.New(t.config, manager.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: "0", - }, - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - options.Cache.DisableFor = []client.Object{ - &corev1.Event{}, - &eventsv1beta1.Event{}, - &eventsv1.Event{}, - } - cl, err := client.New(config, options) - if err != nil { - return nil, err - } - testCl := clientBuilder.WithClient(cl).Build() - return testCl, nil - }, - }) - t.g.Expect(err).ToNot(HaveOccurred()) - t.mgr = mgr - t.client = mgr.GetClient() - ctrl.SetLogger(logr.Discard()) + return nil } From dddba457989fef70d6a7337ed0515ca0a22513b0 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Apr 2024 11:03:01 +0530 Subject: [PATCH 143/235] corrected test-integration target dependencies --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b49742125..3326eb6b3 100644 --- a/Makefile +++ b/Makefile @@ -131,7 +131,7 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) @bash $(HACK_DIR)/e2e-test/run-e2e-test.sh $(PROVIDERS) .PHONY: test-integration -test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTEST) +test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test.sh" ./test/integration/... @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test-go.sh" ./test/it/... From 492fc95b542109fe847dfeaedefedee5d8279039 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Apr 2024 11:08:41 +0530 Subject: [PATCH 144/235] removed reference to gotest from test-go.sh --- hack/test-go.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/test-go.sh b/hack/test-go.sh index e85ee9b08..650ef3ffb 100755 --- a/hack/test-go.sh +++ b/hack/test-go.sh @@ -38,4 +38,4 @@ if ${TEST_COV:-false}; then exit 0 fi -gotest -v "$@" +go test -json "$@" | gotestfmt -hide empty-packages From c6d6a5f5f4fbf53b5d165fd042a05fe045974524 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Apr 2024 11:25:04 +0530 Subject: [PATCH 145/235] changed the version of kustomize to 5.3.0 --- hack/tools.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools.mk b/hack/tools.mk index 98f94c09a..9b8b21120 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -9,7 +9,7 @@ GOTESTFMT := $(TOOLS_BIN_DIR)/gotestfmt # default tool versions SKAFFOLD_VERSION ?= v1.38.0 -KUSTOMIZE_VERSION ?= v4.5.7 +KUSTOMIZE_VERSION ?= v5.3.0 GOTESTFMT_VERSION ?= v2.5.0 export TOOLS_BIN_DIR := $(TOOLS_BIN_DIR) From 440ff6521435ba1a56220a5ede745075ccab76e1 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 22 Apr 2024 13:11:21 +0530 Subject: [PATCH 146/235] Export kind cluster logs to prow job artifacts for enhanced debugging --- hack/ci-e2e-kind.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hack/ci-e2e-kind.sh b/hack/ci-e2e-kind.sh index 200494bef..c0fa3b431 100755 --- a/hack/ci-e2e-kind.sh +++ b/hack/ci-e2e-kind.sh @@ -9,9 +9,10 @@ set -o pipefail make kind-up -trap " - ( make kind-down ) -" EXIT +trap '{ + kind export logs "${ARTIFACTS:-}/etcd-druid-e2e" --name etcd-druid-e2e || true + make kind-down +}' EXIT kubectl wait --for=condition=ready node --all export AWS_APPLICATION_CREDENTIALS_JSON="/tmp/aws.json" From 22cc70eb1f8b67c728fbb2efc1ad527b647ef227 Mon Sep 17 00:00:00 2001 From: Seshachalam Yerasala Venkata Date: Mon, 22 Apr 2024 13:29:47 +0530 Subject: [PATCH 147/235] debug --- hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh b/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh index 428e5beeb..b2bdc8730 100644 --- a/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh +++ b/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh @@ -3,6 +3,7 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors # # SPDX-License-Identifier: Apache-2.0 +set -xe if [[ -n "${LOCALSTACK_HOST}" ]]; then export AWS_ENDPOINT_URL_S3="http://${LOCALSTACK_HOST}" From 926f78a00dae55d36cf57a50383b71bb55704dd7 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Apr 2024 13:55:59 +0530 Subject: [PATCH 148/235] corrected the kustomize version --- hack/tools.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/tools.mk b/hack/tools.mk index 9b8b21120..f48038068 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -9,7 +9,7 @@ GOTESTFMT := $(TOOLS_BIN_DIR)/gotestfmt # default tool versions SKAFFOLD_VERSION ?= v1.38.0 -KUSTOMIZE_VERSION ?= v5.3.0 +KUSTOMIZE_VERSION ?= v4.5.7 GOTESTFMT_VERSION ?= v2.5.0 export TOOLS_BIN_DIR := $(TOOLS_BIN_DIR) @@ -20,7 +20,7 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) ######################################### $(KUSTOMIZE): - @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION} + @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v4@${KUSTOMIZE_VERSION} $(GOTESTFMT): GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@$(GOTESTFMT_VERSION) From 83bac3fee686aa0fd8bd1b7198653be2582259f6 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 22 Apr 2024 14:27:41 +0530 Subject: [PATCH 149/235] Ignore not found debug pods in e2e tests during cleanup --- test/e2e/etcd_backup_test.go | 2 +- test/e2e/etcd_compaction_test.go | 2 +- test/e2e/etcd_multi_node_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index b4364e3b0..fab74d4a0 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -73,7 +73,7 @@ var _ = Describe("Etcd Backup", func() { By("Delete debug pod") etcd := getDefaultEtcd(etcdName, namespace, storageContainer, storePrefix, provider) debugPod := getDebugPod(etcd) - Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(cl.Delete(ctx, debugPod))).ToNot(HaveOccurred()) By("Purge etcd") purgeEtcd(ctx, cl, providers) diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index f19735719..4cc7eee71 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -67,7 +67,7 @@ var _ = Describe("Etcd Compaction", func() { By("Delete debug pod") etcd := getDefaultEtcd(etcdName, namespace, storageContainer, storePrefix, provider) debugPod := getDebugPod(etcd) - Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(cl.Delete(ctx, debugPod))).ToNot(HaveOccurred()) By("Purge etcd") purgeEtcd(ctx, cl, providers) diff --git a/test/e2e/etcd_multi_node_test.go b/test/e2e/etcd_multi_node_test.go index df7701e1b..6d6386583 100644 --- a/test/e2e/etcd_multi_node_test.go +++ b/test/e2e/etcd_multi_node_test.go @@ -129,7 +129,7 @@ var _ = Describe("Etcd", func() { checkEtcdReady(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) By("Delete debug pod") - Expect(cl.Delete(ctx, debugPod)).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(cl.Delete(ctx, debugPod))).ToNot(HaveOccurred()) By("Delete etcd") deleteAndCheckEtcd(ctx, cl, objLogger, etcd, multiNodeEtcdTimeout) From cfb6fec55d9f9bbb0b5766c6176f8d43ac562da0 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 22 Apr 2024 14:28:00 +0530 Subject: [PATCH 150/235] Update etcd-custom-image to v3.4.26-7 --- charts/images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/images.yaml b/charts/images.yaml index a91b6815a..244d693c0 100644 --- a/charts/images.yaml +++ b/charts/images.yaml @@ -8,7 +8,7 @@ images: - name: etcd sourceRepository: github.com/gardener/etcd-custom-image repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcd - tag: "v3.4.26-3" + tag: "v3.4.26-7" - name: etcd-backup-restore-distroless resourceId: name: 'etcdbrctl' From 6c99c2fe34f51d1098a92b5854aa877b0f1d04bb Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 22 Apr 2024 21:01:44 +0530 Subject: [PATCH 151/235] Add documentation for updated controller design, sentinel webhook, CLI flags --- docs/concepts/controllers.md | 55 ++++++++++--------- docs/concepts/webhooks.md | 36 ++++++++++++ docs/deployment/cli-flags.md | 31 +++++++++++ ...m-permanent-quorum-loss-in-etcd-cluster.md | 19 +++++-- internal/controller/compaction/config.go | 4 +- internal/controller/config.go | 2 +- 6 files changed, 112 insertions(+), 35 deletions(-) create mode 100644 docs/concepts/webhooks.md create mode 100644 docs/deployment/cli-flags.md diff --git a/docs/concepts/controllers.md b/docs/concepts/controllers.md index 1cfc3a2c9..539255774 100644 --- a/docs/concepts/controllers.md +++ b/docs/concepts/controllers.md @@ -4,22 +4,21 @@ etcd-druid is an operator to manage etcd clusters, and follows the [`Operator`]( It makes use of the [Kubebuilder](https://github.com/kubernetes-sigs/kubebuilder) framework which makes it quite easy to define Custom Resources (CRs) such as `Etcd`s and `EtcdCopyBackupTask`s through [*Custom Resource Definitions*](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) (CRDs), and define controllers for these CRDs. etcd-druid uses Kubebuilder to define the `Etcd` CR and its corresponding controllers. -All controllers that are a part of etcd-druid reside in package `./controllers`, as sub-packages. +All controllers that are a part of etcd-druid reside in package `internal/controller`, as sub-packages. etcd-druid currently consists of 5 controllers, each having its own responsibility: -- *etcd* : responsible for the reconciliation of the `Etcd` CR, which allows users to run etcd clusters within the specified Kubernetes cluster. -- *custodian* : responsible for the updation of the status of the `Etcd` CR. +- *etcd* : responsible for the reconciliation of the `Etcd` CR spec, which allows users to run etcd clusters within the specified Kubernetes cluster, and also responsible for periodically updating the `Etcd` CR status with the up-to-date state of the managed etcd cluster. - *compaction* : responsible for [snapshot compaction](/docs/proposals/02-snapshot-compaction.md). - *etcdcopybackupstask* : responsible for the reconciliation of the `EtcdCopyBackupsTask` CR, which helps perform the job of copying snapshot backups from one object store to another. - *secret* : responsible in making sure `Secret`s being referenced by `Etcd` resources are not deleted while in use. ## Package Structure -The typical package structure for the controllers that are part of etcd-druid is shown with the *custodian controller*: +The typical package structure for the controllers that are part of etcd-druid is shown with the *compaction controller*: ``` bash -controllers/custodian +internal/controller/compaction ├── config.go ├── reconciler.go └── register.go @@ -39,42 +38,46 @@ Once the manager is `Start()`ed, all the controllers that are *registered* with Each controller is built using a controller builder, configured with details such as the type of object being reconciled, owned objects whose owner object is reconciled, event filters (predicates), etc. `Predicates` are filters which allow controllers to filter which type of events the controller should respond to and which ones to ignore. -The logic relevant to the controller manager like the creation of the controller manager and registering each of the controllers with the manager, is contained in [`controllers/manager.go`](/controllers/manager.go). +The logic relevant to the controller manager like the creation of the controller manager and registering each of the controllers with the manager, is contained in [`internal/controller/manager.go`](/internal/controller/manager.go). ## Etcd Controller -The *etcd controller* is responsible for the reconciliation of the `Etcd` resource. -It handles the provisioning and management of the etcd cluster. Different components that are required for the functioning of the cluster like `Leases`, `ConfigMap`s, and the `Statefulset` for the etcd cluster are all deployed and managed by the *etcd controller*. +The *etcd controller* is responsible for the reconciliation of the `Etcd` resource spec and status. It handles the provisioning and management of the etcd cluster. Different components that are required for the functioning of the cluster like `Leases`, `ConfigMap`s, and the `Statefulset` for the etcd cluster are all deployed and managed by the *etcd controller*. -While building the controller, an event filter is set such that the behavior of the controller depends on the `gardener.cloud/operation: reconcile` *annotation*. This is controlled by the `--ignore-operation-annotation` CLI flag, which, if set to `false`, tells the controller to perform reconciliation only when this annotation is present. If the flag is set to `true`, the controller will trigger reconciliation anytime the `Etcd` spec, and thus `generation`, changes. +Additionally, *etcd controller* also periodically updates the `Etcd` resource status with latest available information from the etcd cluster, as well as results and errors from the recentmost reconciliation of the `Etcd` resource spec. -The reason this filter is present is that any disruption in the `Etcd` resource due to reconciliation (due to changes in the `Etcd` spec, for example) while workloads are being run would be disastrous. -Hence, any user who wishes to avoid such disruptions, can choose to set the `--ignore-operation-annotation` CLI flag to `false`. An example of this is Gardener's [gardenlet](https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md), which reconciles the `Etcd` resource only during a shoot cluster's [*maintenance window*](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_maintenance.md). +The *etcd controller* is essential to the functioning of the etcd cluster and etcd-druid, thus the minimum number of worker threads is 1 (default being 3). -The controller adds a finalizer to the `Etcd` resource in order to ensure that the `Etcd` instance does not get deleted while the system is still dependent on the existence of the `Etcd` resource. -Only the *etcd controller* can delete a resource once it adds finalizers to it. This ensures that the proper deletion flow steps are followed while deleting the resource. When the *etcd controller* enters the deletion flow, components are deleted in the reverse order that they were deployed in. +### `Etcd` Spec Reconciliation -The *etcd controller* is essential to the functioning of the etcd cluster and etcd-druid, thus the minimum number of worker threads is 1 (default being 3). +While building the controller, an event filter is set such that the behavior of the controller depends on the `gardener.cloud/operation: reconcile` *annotation*. This is controlled by the `--enable-etcd-spec-auto-reconcile` CLI flag, which if set to `false`, tells the controller to perform reconciliation only when this annotation is present. If the flag is set to `true`, the controller will reconcile the etcd cluster anytime the `Etcd` spec, and thus `generation`, changes, and the next queued event for it is triggered. -## Custodian Controller +The reason this filter is present is that any disruption in the `Etcd` resource due to reconciliation (due to changes in the `Etcd` spec, for example) while workloads are being run would cause unwanted downtimes to the etcd cluster. Hence, any user who wishes to avoid such disruptions, can choose to set the `--enable-etcd-spec-auto-reconcile` CLI flag to `false`. An example of this is Gardener's [gardenlet](https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md), which reconciles the `Etcd` resource only during a shoot cluster's [*maintenance window*](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_maintenance.md). -The *custodian controller* acts on the `Etcd` resource. -The primary purpose of the *custodian controller* is to update the status of the `Etcd` resource. +The controller adds a finalizer to the `Etcd` resource in order to ensure that it does not get deleted until all dependent resources managed by druid, aka managed components, are properly cleaned up. Only the *etcd controller* can delete a resource once it adds finalizers to it. This ensures that the proper deletion flow steps are followed while deleting the resource. During deletion flow, managed components are deleted in parallel. -It watches for changes in the status of the `Statefulset`s associated with the `Etcd` resources. -Even though the `Etcd` resource owns the `Statefulset`, it is not necessary that the *etcd controller* reconciles whenever there are changes in the statuses of the objects that the `Etcd` resource owns. +### `Etcd` Status Updates -Status fields of the `Etcd` resource which correspond to the `StatefulSet` like `CurrentReplicas`, `ReadyReplicas`, `Replicas` and `Ready` are updated to reflect those of the `StatefulSet` by the controller. Cluster membership (`EtcdMemberStatus`) and `Conditions` are updated as follows: +The `Etcd` resource status is updated periodically by `etcd controller`, the interval for which is determined by the CLI flag `--etcd-status-sync-period`. -- Cluster Membership: The controller updates the information about etcd cluster membership like `Role`, `Status`, `Reason`, `LastTransitionTime` and identifying information like the `Name` and `ID`. For the `Status` field, the member is checked for the *Ready* condition, where the member can be in `Ready`, `NotReady` and `Unknown` statuses. +Status fields of the `Etcd` resource such as `LastOperation`, `LastErrors` and `ObservedGeneration`, are updated to reflect the result of the recent reconciliation of the `Etcd` resource spec. -- Condition: The controller updates the `Conditions` field which holds the latest information of the `Etcd`'s state. The condition checks that are performed are `AllMembersCheck`, `ReadyCheck` and `BackupReadyCheck`. The first two of these checks make use of the `EtcdMemberStatus` to update `Conditions` with information about the members, and the last corresponds to the status of the backup. +- `LastOperation` holds information about the last operation performed on the etcd cluster, indicated by fields `Type`, `State`, `Description` and `LastUpdateTime`. Additionally, a field `RunID` indicates the unique ID assigned to the specific reconciliation run, to allow for better debugging of issues. +- `LastErrors` is a slice of errors encountered by the last reconciliation run. Each error consists of fields `Code` to indicate the custom druid error code for the error, a human-readable `Description`, and the `ObservedAt` time when the error was seen. +- `ObservedGeneration` indicates the latest `generation` of the `Etcd` resource that druid has "observed" and consequently reconciled. It helps identify whether a change in the `Etcd` resource spec was acted upon by druid or not. -To reflect changes that occur in the `Statefulset` status in the `Etcd` resource, the *custodian controller* keeps a watch on the `Statefulset`. +Status fields of the `Etcd` resource which correspond to the `StatefulSet` like `CurrentReplicas`, `ReadyReplicas` and `Replicas` are updated to reflect those of the `StatefulSet` by the controller. + +Status fields related to the etcd cluster itself, such as `Members`, `PeerUrlTLSEnabled` and `Ready` are updated as follows: + +- Cluster Membership: The controller updates the information about etcd cluster membership like `Role`, `Status`, `Reason`, `LastTransitionTime` and identifying information like the `Name` and `ID`. For the `Status` field, the member is checked for the *Ready* condition, where the member can be in `Ready`, `NotReady` and `Unknown` statuses. -The *custodian controller* reconciles periodically, which can be set through the `--custodian-sync-period` CLI flag (default being 30 seconds). It also reconciles whenever there are changes to the `Statefulset` status. +`Etcd` resource conditions are indicated by status field `Conditions`. The condition checks that are currently performed are: -The *custodian controller* is essential to the functioning of etcd-druid, thus the minimum number of worker threads is 1, (default being 3). +- `AllMembersReady`: indicates readiness of all members of the etcd cluster. +- `Ready`: indicates overall readiness of the etcd cluster in serving traffic. +- `BackupReady`: indicates health of the etcd backups, ie, whether etcd backups are being taken regularly as per schedule. This condition is applicable only when backups are enabled for the etcd cluster. +- `DataVolumesReady`: indicates health of the persistent volumes containing the etcd data. ## Compaction Controller @@ -88,7 +91,7 @@ The number of worker threads for the *compaction controller* needs to be greater This is unlike other controllers which need at least one worker thread for the proper functioning of etcd-druid as snapshot compaction is not a core functionality for the etcd clusters to be deployed. The compaction controller should be explicitly enabled by the user, through the `--enable-backup-compaction` CLI flag. -## Etcdcopybackupstask Controller +## EtcdCopyBackupsTask Controller The *etcdcopybackupstask controller* is responsible for deploying the [`etcdbrctl copy`](https://github.com/gardener/etcd-backup-restore/blob/master/cmd/copy.go) command as a job. This controller reacts to create/update events arising from EtcdCopyBackupsTask resources, and deploys the `EtcdCopyBackupsTask` job with source and target backup storage providers as arguments, which are derived from source and target bucket secrets referenced by the `EtcdCopyBackupsTask` resource. diff --git a/docs/concepts/webhooks.md b/docs/concepts/webhooks.md new file mode 100644 index 000000000..3490633df --- /dev/null +++ b/docs/concepts/webhooks.md @@ -0,0 +1,36 @@ +# Webhooks + +The [etcd-druid controller-manager](controllers.md#controller-manager) registers certain [admission webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) that allow for validation or mutation of requests on resources in the cluster, in order to prevent misconfiguration and restrict access to the etcd cluster resources. + +All webhooks that are a part of etcd-druid reside in package `internal/webhook`, as sub-packages. + +## Package Structure + +The typical package structure for the webhooks that are part of etcd-druid is shown with the *sentinel webhook*: + +``` bash +internal/webhook/sentinel +├── config.go +├── handler.go +└── register.go +``` + +- `config.go`: contains all the logic for the configuration of the webhook, including feature gate activations, CLI flag parsing and validations. +- `register.go`: contains the logic for registering the webhook with the etcd-druid controller manager. +- `handler.go`: contains the webhook admission handler logic. + +Each webhook package may also contains auxiliary files which are relevant to that specific webhook. + +## Sentinel Webhook + +Druid controller-manager registers and runs the [etcd controller](controllers.md#etcd-controller), which creates and manages various resources such as `Leases`, `ConfigMap`s, and the `Statefulset` for the etcd cluster. It is essential for all these resources to contain correct configuration for the proper functioning of the etcd cluster. + +Unintended changes to any of these *managed resources* can lead to misconfiguration of the etcd cluster, leading to unwanted downtime for etcd traffic. To prevent such unintended changes, a validating webhook called *sentinel webhook* guards these managed resources, ensuring that only authorized entities can perform operations on these managed resources. + +*Sentinel webhook* prevents *UPDATE* and *DELETE* operations on all resources managed by *etcd controller*, unless such an operation is performed by druid itself, and during reconciliation of the `Etcd` resource. Operations are also allowed if performed by one of the authorized entities specified by CLI flag `--sentinel-exempt-service-accounts`. + +There may be specific cases where a human operator may need to make changes to the managed resources, possibly to test or fix an etcd cluster. An example of this is [recovery from permanent quorum loss](../operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md), where a human operator will need to suspend reconciliation of the `Etcd` resource, make changes to the underlying managed resources such as `StatefulSet` and `ConfigMap`, and then resume reconciliation for the `Etcd` resource. Such manual interventions will require out-of-band changes to the managed resources. Protection of managed resources for such `Etcd` resources can be turned off by adding an annotation `druid.gardener.cloud/resource-protection: false` on the `Etcd` resource. This will effectively disable *sentinel webhook* protection for all managed resources for the specific `Etcd`. + +**Note:** *UPDATE* operations for `Lease`s are always allowed, since these are regularly updated by the etcd-backup-restore sidecar. + +The *sentinel webhook* is disabled by default, and can be enabled via the CLI flag `--enable-sentinel-webhook`. diff --git a/docs/deployment/cli-flags.md b/docs/deployment/cli-flags.md new file mode 100644 index 000000000..e58be11d0 --- /dev/null +++ b/docs/deployment/cli-flags.md @@ -0,0 +1,31 @@ +# CLI Flags + +Etcd-druid exposes the following CLI flags that allow for configuring its behavior. + +| CLI FLag | Component | Description | Default | +|-----------------------------------------|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------| +| `metrics-addr` | `controller-manager` | The address the metric endpoint binds to. | `":8080"` | +| `webhook-server-bind-address` | `controller-manager` | The IP address on which to listen for the HTTPS webhook server. | `""` | +| `webhook-server-port` | `controller-manager` | The port on which to listen for the HTTPS webhook server. | `9443` | +| `webhook-server-tls-server-cert-dir` | `controller-manager` | The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively). | `"/etc/webhook-server-tls"` | +| `enable-leader-election` | `controller-manager` | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. | `false` | +| `leader-election-id` | `controller-manager` | Name of the resource that leader election will use for holding the leader lock. | `"druid-leader-election"` | +| `leader-election-resource-lock` | `controller-manager` | Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.
Deprecated. Will be removed in the future in favour of using only `leases` as the leader election resource lock for the controller manager. | `"leases"` | +| `disable-lease-cache` | `controller-manager` | Disable cache for lease.coordination.k8s.io resources. | `false` | +| `etcd-workers` | `etcd-controller` | Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed. | `3` | +| `ignore-operation-annotation` | `etcd-controller` | Specifies whether to ignore or honour the operation annotation on resources to be reconciled.
Deprecated. Use `--enable-etcd-spec-auto-reconcile` instead. | `false` | +| `enable-etcd-spec-auto-reconcile` | `etcd-controller` | If true then automatically reconciles Etcd Spec. If false waits for explicit annotation to be placed on the Etcd resource to trigger reconcile. | `false` | +| `disable-etcd-serviceaccount-automount` | `etcd-controller` | If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets. | `false` | +| `etcd-status-sync-period` | `etcd-controller` | Period after which an etcd status sync will be attempted. | `15s` | +| `etcd-member-notready-threshold` | `etcd-controller` | Threshold after which an etcd member is considered not ready if the status was unknown before. | `5m` | +| `etcd-member-unknown-threshold` | `etcd-controller` | Threshold after which an etcd member is considered unknown. | `1m` | +| `enable-backup-compaction` | `compaction-controller` | Enable automatic compaction of etcd backups. | `false` | +| `compaction-workers` | `compaction-controller` | Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. Setting this flag to 0 disables the controller. | `3` | +| `etcd-events-threshold` | `compaction-controller` | Total number of etcd events that can be allowed before a backup compaction job is triggered. | `1000000` | +| `active-deadline-duration` | `compaction-controller` | Duration after which a running backup compaction job will be terminated. | `3h` | +| `metrics-scrape-wait-duration` | `compaction-controller` | Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped. | `0s` | +| `etcd-copy-backups-task-workers` | `etcdcopybackupstask-controller` | Number of worker threads for the etcdcopybackupstask controller. | `3` | +| `secret-workers` | `secret-controller` | Number of worker threads for the secrets controller. | `10` | +| `enable-sentinel-webhook` | `sentinel-webhook` | Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid. | `false` | +| `reconciler-service-account` | `sentinel-webhook` | The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. | `` | +| `sentinel-exempt-service-accounts` | `sentinel-webhook` | The comma-separated list of fully qualified names of service accounts that are exempt from Sentinel Webhook checks. | `""` | diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md index f20f0dfe5..fa73c4a20 100644 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -20,9 +20,12 @@ There are two [Etcd clusters](https://github.com/gardener/etcd-druid/tree/master Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. - 1. Add the following annotation to the `Etcd` resource `kubectl annotate etcd etcd-main druid.gardener.cloud/ignore-reconciliation="true"` - - 2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: + 1. Add the following annotations to the `Etcd` resource `etcd-main`: + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile="true"` + + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection="false"` + + 2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: ``` Volumes: @@ -50,8 +53,8 @@ There are two [Etcd clusters](https://github.com/gardener/etcd-druid/tree/master `kubectl delete pvc -l instance=etcd-main` 5. Check the etcd's member leases. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. - One of those leases will have holder identity as `:Leader` and rest of etcd member leases have holder identities as `:Member`. - Please ignore the snapshot leases i.e those leases which have suffix `snap`. + One of those leases will have holder identity as `:Leader` and rest of etcd member leases have holder identities as `:Member`. + Please ignore the snapshot leases i.e those leases which have suffix `snap`. etcd-main member leases: ``` @@ -90,7 +93,11 @@ There are two [Etcd clusters](https://github.com/gardener/etcd-druid/tree/master etcd-main-0 2/2 Running 0 1m ``` - 9. Remove the following annotation from the `Etcd` resource `etcd-main`: `kubectl annotate etcd etcd-main druid.gardener.cloud/ignore-reconciliation-` + 9. Remove the following annotations from the `Etcd` resource `etcd-main`: + + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile-` + + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection-` 10. Finally add the following annotation to the `Etcd` resource `etcd-main`: `kubectl annotate etcd etcd-main gardener.cloud/operation="reconcile"` diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 474a3a086..087510efa 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -58,9 +58,9 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.Int64Var(&cfg.EventsThreshold, eventsThresholdFlagName, defaultEventsThreshold, "Total number of etcd events that can be allowed before a backup compaction job is triggered.") fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, - "Duration after which a running backup compaction job will be killed (Ex: \"300ms\", \"20s\" or \"2h45m\").\").") + "Duration after which a running backup compaction job will be terminated.") fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagname, defaultMetricsScrapeWaitDuration, - "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped (Ex: \"300ms\", \"60s\" or \"2h45m\").\").") + "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped.") } // Validate validates the config. diff --git a/internal/controller/config.go b/internal/controller/config.go index e4d6213d7..8abb96088 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -125,7 +125,7 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { flag.BoolVar(&cfg.EnableLeaderElection, enableLeaderElectionFlagName, defaultEnableLeaderElection, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") flag.StringVar(&cfg.LeaderElectionID, leaderElectionIDFlagName, defaultLeaderElectionID, - "Name of the resource that leader election will use for holding the leader lock") + "Name of the resource that leader election will use for holding the leader lock.") flag.StringVar(&cfg.LeaderElectionResourceLock, leaderElectionResourceLockFlagName, defaultLeaderElectionResourceLock, "Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.") flag.BoolVar(&cfg.DisableLeaseCache, disableLeaseCacheFlagName, defaultDisableLeaseCache, From 4ba67b9be1ec3a29ce25a7d61aeec8141fecf0be Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 23 Apr 2024 13:24:19 +0530 Subject: [PATCH 152/235] fix for issue #759 --- internal/operator/statefulset/builder.go | 7 ------- internal/operator/statefulset/stsmatcher.go | 5 ----- 2 files changed, 12 deletions(-) diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index c93479b73..bdc36b168 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -388,13 +388,6 @@ func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { Env: env, Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Backup.Resources, defaultResourceRequirements), VolumeMounts: b.getBackupRestoreContainerVolumeMounts(), - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_PTRACE", - }, - }, - }, }, nil } diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 069b16e8c..926263673 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -255,11 +255,6 @@ func (s StatefulSetMatcher) matchBackupRestoreContainer() gomegatypes.GomegaMatc ), "Resources": Equal(containerResources), "VolumeMounts": s.matchBackupRestoreContainerVolMounts(), - "SecurityContext": PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ - "Capabilities": PointTo(MatchFields(IgnoreExtras, Fields{ - "Add": ConsistOf(corev1.Capability("SYS_PTRACE")), - })), - })), }) } From 6eaf7ee19c1e8abc1ca2a4a8be349a944c7cc702 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 3 May 2024 22:42:16 +0530 Subject: [PATCH 153/235] Address review comments by @renormalize; Make all CLI flags configurable in helm charts --- Makefile | 4 +- api/v1alpha1/types_etcd.go | 38 ++++---- .../templates/controller-deployment.yaml | 89 +++++++++++-------- charts/druid/templates/druid-clusterrole.yaml | 2 +- charts/druid/templates/druid-rolebinding.yaml | 2 +- charts/druid/templates/secret-ca-crt.yaml | 2 +- .../templates/secret-server-tls-crt.yaml | 2 +- charts/druid/templates/service-account.yaml | 2 +- charts/druid/templates/service.yaml | 18 ++-- .../templates/validating-webhook-config.yaml | 6 +- charts/druid/values.yaml | 68 ++++++++++---- docs/concepts/controllers.md | 12 +-- docs/concepts/webhooks.md | 2 +- docs/deployment/cli-flags.md | 7 +- .../overlays/aws/common/files/common.sh | 2 +- internal/controller/config.go | 14 ++- internal/controller/manager.go | 8 ++ skaffold.yaml | 13 +-- 18 files changed, 184 insertions(+), 107 deletions(-) diff --git a/Makefile b/Makefile index 3326eb6b3..dc4f4714d 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ docker-push: # Run tests .PHONY: test test: $(GINKGO) $(GOTESTFMT) - @# run ginkgo unit tests. These will be ported to golang native tests over a period of time. + # run ginkgo unit tests. These will be ported to golang native tests over a period of time. @"$(REPO_ROOT)/hack/test.sh" ./api/... \ ./internal/controller/etcdcopybackupstask/... \ ./internal/controller/predicate/... \ @@ -115,7 +115,7 @@ test: $(GINKGO) $(GOTESTFMT) ./internal/controller/utils/... \ ./internal/mapper/... \ ./internal/metrics/... - @# run the golang native unit tests. + # run the golang native unit tests. @TEST_COV="true" "$(REPO_ROOT)/hack/test-go.sh" ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... .PHONY: test-cov diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 23c87198d..51200dec0 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -193,7 +193,7 @@ type BackupSpec struct { LeaderElection *LeaderElectionSpec `json:"leaderElection,omitempty"` } -// EtcdConfig defines parameters associated etcd deployed +// EtcdConfig defines the configuration for the etcd cluster to be deployed. type EtcdConfig struct { // Quota defines the etcd DB quota. // +optional @@ -250,7 +250,7 @@ type SharedConfig struct { // AutoCompactionMode defines the auto-compaction-mode:'periodic' mode or 'revision' mode for etcd and embedded-Etcd of backup-restore sidecar. // +optional AutoCompactionMode *CompactionMode `json:"autoCompactionMode,omitempty"` - //AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. + // AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-etcd of backup-restore sidecar. // +optional AutoCompactionRetention *string `json:"autoCompactionRetention,omitempty"` } @@ -332,11 +332,11 @@ const ( type EtcdMemberConditionStatus string const ( - // EtcdMemberStatusReady means a etcd member is ready. + // EtcdMemberStatusReady indicates that the etcd member is ready. EtcdMemberStatusReady EtcdMemberConditionStatus = "Ready" - // EtcdMemberStatusNotReady means a etcd member is not ready. + // EtcdMemberStatusNotReady indicates that the etcd member is not ready. EtcdMemberStatusNotReady EtcdMemberConditionStatus = "NotReady" - // EtcdMemberStatusUnknown means the status of an etcd member is unknown. + // EtcdMemberStatusUnknown indicates that the status of the etcd member is unknown. EtcdMemberStatusUnknown EtcdMemberConditionStatus = "Unknown" ) @@ -350,7 +350,7 @@ const ( EtcdRoleMember EtcdRole = "Member" ) -// EtcdMemberStatus holds information about a etcd cluster membership. +// EtcdMemberStatus holds information about etcd cluster membership. type EtcdMemberStatus struct { // Name is the name of the etcd member. It is the name of the backing `Pod`. Name string `json:"name"` @@ -399,7 +399,7 @@ type EtcdStatus struct { // CurrentReplicas is the current replica count for the etcd cluster. // +optional CurrentReplicas int32 `json:"currentReplicas,omitempty"` - // Replicas is the replica count of the etcd resource. + // Replicas is the replica count of the etcd cluster. // +optional Replicas int32 `json:"replicas,omitempty"` // ReadyReplicas is the count of replicas being ready in the etcd cluster. @@ -429,11 +429,11 @@ type EtcdStatus struct { type LastOperationType string const ( - // LastOperationTypeCreate indicates that the last operation was a creation of a new etcd resource. + // LastOperationTypeCreate indicates that the last operation was a creation of a new Etcd resource. LastOperationTypeCreate LastOperationType = "Create" - // LastOperationTypeReconcile indicates that the last operation was a reconciliation of the spec of an etcd resource. + // LastOperationTypeReconcile indicates that the last operation was a reconciliation of the spec of an Etcd resource. LastOperationTypeReconcile LastOperationType = "Reconcile" - // LastOperationTypeDelete indicates that the last operation was a deletion of an existing etcd resource. + // LastOperationTypeDelete indicates that the last operation was a deletion of an existing Etcd resource. LastOperationTypeDelete LastOperationType = "Delete" ) @@ -458,10 +458,10 @@ type LastOperation struct { // Description describes the last operation. Description string `json:"description"` // RunID correlates an operation with a reconciliation run. - // Every time an etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is + // Every time an Etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is // generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this // as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering - // reconcile logs as all structured logs in a reconcile run should have the `runID` referenced. + // reconcile logs as all structured logs in a reconciliation run should have the `runID` referenced. RunID string `json:"runID"` // LastUpdateTime is the time at which the operation was updated. LastUpdateTime metav1.Time `json:"lastUpdateTime"` @@ -480,7 +480,7 @@ type LastError struct { ObservedAt metav1.Time `json:"observedAt"` } -// GetNamespaceName is a convenience function which creates a types.NamespacedName for an etcd resource. +// GetNamespaceName is a convenience function which creates a types.NamespacedName for an Etcd resource. func (e *Etcd) GetNamespaceName() types.NamespacedName { return types.NamespacedName{ Namespace: e.Namespace, @@ -568,7 +568,7 @@ func (e *Etcd) GetRoleBindingName() string { return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, e.Name) } -// IsBackupStoreEnabled returns true if backup store has been enabled for this etcd, else returns false. +// IsBackupStoreEnabled returns true if backup store has been enabled for the Etcd resource, else returns false. func (e *Etcd) IsBackupStoreEnabled() bool { return e.Spec.Backup.Store != nil } @@ -578,8 +578,8 @@ func (e *Etcd) IsMarkedForDeletion() bool { return !e.DeletionTimestamp.IsZero() } -// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an etcd resource signalling the intent -// to suspend spec reconciliation for this etcd resource. If no annotation is set then it will return nil. +// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an Etcd resource signalling the intent +// to suspend spec reconciliation for this Etcd resource. If no annotation is set then it will return nil. func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { if metav1.HasAnnotation(e.ObjectMeta, SuspendEtcdSpecReconcileAnnotation) { return pointer.String(SuspendEtcdSpecReconcileAnnotation) @@ -590,14 +590,14 @@ func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { return nil } -// IsReconciliationSuspended returns true if the etcd resource has the annotation set to suspend spec reconciliation, +// IsReconciliationSuspended returns true if the Etcd resource has the annotation set to suspend spec reconciliation, // else returns false. func (e *Etcd) IsReconciliationSuspended() bool { suspendReconcileAnnotKey := e.GetSuspendEtcdSpecReconcileAnnotationKey() return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) } -// AreManagedResourcesProtected returns true if the etcd resource has the resource protection annotation set to true, +// AreManagedResourcesProtected returns true if the Etcd resource has the resource protection annotation set to true, // else returns false. func (e *Etcd) AreManagedResourcesProtected() bool { if metav1.HasAnnotation(e.ObjectMeta, ResourceProtectionAnnotation) { @@ -606,7 +606,7 @@ func (e *Etcd) AreManagedResourcesProtected() bool { return true } -// IsReconciliationInProgress returns true if the etcd resource is currently being reconciled, else returns false. +// IsReconciliationInProgress returns true if the Etcd resource is currently being reconciled, else returns false. func (e *Etcd) IsReconciliationInProgress() bool { return e.Status.LastOperation != nil && (e.Status.LastOperation.State == LastOperationStateProcessing || diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index 0e66fa082..1f29a21fa 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -5,16 +5,16 @@ metadata: name: etcd-druid namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid spec: replicas: {{ .Values.replicas }} selector: matchLabels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid template: metadata: labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid spec: serviceAccountName: etcd-druid containers: @@ -23,21 +23,6 @@ spec: imagePullPolicy: {{ .Values.image.imagePullPolicy }} command: - /etcd-druid - - --enable-leader-election=true - - --ignore-operation-annotation={{ .Values.ignoreOperationAnnotation }} - - --etcd-workers=3 - {{- if .Values.etcdStatusSyncPeriod }} - - --etcd-status-sync-period={{ .Values.etcdStatusSyncPeriod }} - {{- end }} - {{- if .Values.enableBackupCompaction }} - - --enable-backup-compaction={{ .Values.enableBackupCompaction }} - {{- end }} - {{- if .Values.eventsThreshold }} - - --etcd-events-threshold={{ .Values.eventsThreshold }} - {{- end }} - {{- if .Values.metricsScrapeWaitDuration }} - - --metrics-scrape-wait-duration={{ .Values.metricsScrapeWaitDuration }} - {{- end }} {{- if .Values.featureGates }} {{- $featuregates := "" }} @@ -47,33 +32,67 @@ spec: - --feature-gates={{ $featuregates | trimSuffix "," }} {{- end }} - {{- if (((.Values.server).webhook).bindAddress) }} - - --webhook-server-bind-address={{ .Values.server.webhook.bindAddress }} + {{- if ((((.Values.controllerManager).server).metrics).bindAddress) }} + - --metrics-bind-address={{ .Values.controllerManager.server.metrics.bindAddress }} + {{- end }} + {{- if ((((.Values.controllerManager).server).metrics).port) }} + - --metrics-port={{ .Values.controllerManager.server.metrics.port }} + {{- end }} + + {{- if ((((.Values.controllerManager).server).webhook).bindAddress) }} + - --webhook-server-bind-address={{ .Values.controllerManager.server.webhook.bindAddress }} + {{- end }} + {{- if ((((.Values.controllerManager).server).webhook).port) }} + - --webhook-server-port={{ .Values.controllerManager.server.webhook.port }} {{- end }} - {{- if (((.Values.server).webhook).port) }} - - --webhook-server-port={{ .Values.server.webhook.port }} + {{- if (((((.Values.controllerManager).server).webhook).tls).serverCertDir) }} + - --webhook-server-tls-server-cert-dir={{ .Values.controllerManager.server.webhook.tls.serverCertDir }} + {{- end }} + + {{- if (((.Values.controllerManager).leaderElection).enabled) }} + - --enable-leader-election={{ .Values.controllerManager.leaderElection.enabled }} + - --leader-election-id={{ .Values.controllerManager.leaderElection.id }} {{- end }} - {{- if ((((.Values.server).webhook).tls).serverCertDir) }} - - --webhook-server-tls-server-cert-dir={{ .Values.server.webhook.tls.serverCertDir }} + + {{- if ((.Values.controllerManager).disableLeaseCache) }} + - --disable-lease-cache={{ .Values.controllerManager.disableLeaseCache }} + {{- end }} + + {{- if ((.Values.controllers).etcd) }} + - --etcd-workers={{ .Values.controllers.etcd.workers }} + - --enable-etcd-spec-auto-reconcile={{ .Values.controllers.etcd.enableEtcdSpecAutoReconcile }} + - --disable-etcd-serviceaccount-automount={{ .Values.controllers.etcd.disableEtcdServiceAccountAutomount }} + - --etcd-status-sync-period={{ .Values.controllers.etcd.etcdStatusSyncPeriod }} + - --etcd-member-notready-threshold={{ .Values.controllers.etcd.etcdMemberNotReadyThreshold }} + - --etcd-member-unknown-threshold={{ .Values.controllers.etcd.etcdMemberUnknownThreshold }} + {{- end }} + + {{- if and ((.Values.controllers).compaction) (eq .Values.controllers.compaction.enabled true) }} + - --enable-backup-compaction=true + - --compaction-workers={{ .Values.controllers.compaction.workers }} + - --etcd-events-threshold={{ int $.Values.controllers.compaction.etcdEventsThreshold }} + - --active-deadline-duration={{ .Values.controllers.compaction.activeDeadlineDuration }} + - --metrics-scrape-wait-duration={{ .Values.controllers.compaction.metricsScrapeWaitDuration }} + {{- end }} + + {{- if (((.Values.controllers).etcdCopyBackupsTask).workers) }} + - --etcd-copy-backups-task-workers={{ .Values.controllers.etcdCopyBackupsTask.workers }} + {{- end }} + + {{- if (((.Values.controllers).secret).workers) }} + - --secret-workers={{ .Values.controllers.secret.workers }} {{- end }} - {{- if (.Values.sentinelWebhook).enabled }} + {{- if ((.Values.webhooks).sentinel).enabled }} - --enable-sentinel-webhook=true - --reconciler-service-account=system:serviceaccount:{{ .Release.Namespace }}:etcd-druid - {{- if .Values.sentinelWebhook.exemptServiceAccounts }} - - --sentinel-exempt-service-accounts={{ join "," .Values.sentinelWebhook.exemptServiceAccounts }} + {{- if .Values.webhooks.sentinel.exemptServiceAccounts }} + - --sentinel-exempt-service-accounts={{ join "," .Values.webhooks.sentinel.exemptServiceAccounts }} {{- end }} {{- end }} resources: - limits: - cpu: 300m - memory: 512Mi - requests: - cpu: 50m - memory: 128Mi - ports: - - containerPort: 9569 +{{ toYaml .Values.resources | indent 10 }} volumeMounts: - mountPath: /etc/webhook-server-tls name: tls diff --git a/charts/druid/templates/druid-clusterrole.yaml b/charts/druid/templates/druid-clusterrole.yaml index 191ffb1d0..886b178da 100644 --- a/charts/druid/templates/druid-clusterrole.yaml +++ b/charts/druid/templates/druid-clusterrole.yaml @@ -5,7 +5,7 @@ metadata: name: gardener.cloud:system:etcd-druid namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid rules: - apiGroups: - "" diff --git a/charts/druid/templates/druid-rolebinding.yaml b/charts/druid/templates/druid-rolebinding.yaml index 4260cd140..ef948bf45 100644 --- a/charts/druid/templates/druid-rolebinding.yaml +++ b/charts/druid/templates/druid-rolebinding.yaml @@ -5,7 +5,7 @@ metadata: name: gardener.cloud:system:etcd-druid namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/charts/druid/templates/secret-ca-crt.yaml b/charts/druid/templates/secret-ca-crt.yaml index ea1096858..9fbea5cd1 100644 --- a/charts/druid/templates/secret-ca-crt.yaml +++ b/charts/druid/templates/secret-ca-crt.yaml @@ -4,7 +4,7 @@ metadata: name: ca-etcd-druid namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid type: Opaque data: bundle.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3ekNDQWRlZ0F3SUJBZ0lKQU51a3lQUVVxYnhNTUEwR0NTcUdTSWIzRFFFQkN3VUFNQlV4RXpBUkJnTlYKQkFNTUNtVjBZMlF0WkhKMWFXUXdIaGNOTWpRd01URTRNVFV6TVRJeldoY05NelF3TVRFNE1UVXpNVEl6V2pBVgpNUk13RVFZRFZRUUREQXBsZEdOa0xXUnlkV2xrTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCCkNnS0NBUUVBeWx1TFQveTl1YmxVdVJ0NDBOUW1xRTVFSWxlakROVTRnREd3WDJ3K2t6RHZ0bFdsYlF0STdQSVoKWTZSK1kvYXF2OHhiOCt3R0tlWEM4bGtSQ1BsZjNIdGpTVUxWUkkzYnVBd29WODJ2YjU4NlQ5Z0Q3OHJKVHdYVwpxcm1kc25VUGR5UzAwSlFQY2d3TzE3QW9DSXl0MkVyclQzWlpyMFpsMVJsUWZoTTc0VVpNbi8xeTJqNUFmUTJPCmh2emdmYnl6ZHZNT0hycnd4NHoxblBsVjNPU3M3WGo3TkM5UDZ4bnBTVTU1MnF1bkZmTDdUc3U3OTQ4NWdZZ3AKOFYrTG9oclY0MWdGWmNlVlh6Y3k1TGhGK093eFBGSG0rMlJlWTZ2UmxmMlU5S2lTTzA2MVBOcy9DYklhWDZucwpmMC9oTVJYbm16c0Q2VU1zd0ZxT0Q4UWw3VTUzQ1FJREFRQUJvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBYVl3CkR3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXFWUG93RXR4akFvT0xtOVl6ZUhIZzBFMmdWb3cKRFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUR3ZGduT25POStWNmNvdTQ4Z1czeUtONVhMNmJkRm5mbng5aldlTQpkelBsc2tBa0hKTzAvTXpqbk1DUnFxQmREajM2Yzcvb3lJcEpBall1ZmFrMXJDcW9rVEpHOVNzTlE4VlJmT0FSCkRRU0o0YmsrMzNQT29MSnNrZTNzdm9ZM0RPc3h4RzZNT0o5S29aRTFaeWxEd3hrYVo0U212b1NmTStmUHU3aloKenA1RWJXVnE0YmJUYjgrRnZqSDQ5bmY1eGY1WHJLbTJ1Z3dZZWIwWnpnYkIrSGdNSHRvMGtWcXFObVlrbDYyMwp0NFBOV3Q1ck9sY2swd1dKeVVZZHdONk1qQWQrZVZaQmJRRzQ3TWpZNTJTenBwbGNQdVJuaU81Q0tKb0JZdmU1CmZzTUhIR0dXeEg0b2JCNk9WZytIM0pJNGpRWndTUXZGRVBKcm5LNXV0RjdySmw0PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== diff --git a/charts/druid/templates/secret-server-tls-crt.yaml b/charts/druid/templates/secret-server-tls-crt.yaml index e4898e247..661228ad5 100644 --- a/charts/druid/templates/secret-server-tls-crt.yaml +++ b/charts/druid/templates/secret-server-tls-crt.yaml @@ -4,7 +4,7 @@ metadata: name: etcd-druid-server-tls namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid type: kubernetes.io/tls data: # TODO: generate TLS server cert that can match etcd-druid.*.svc, since the helm chart can be deployed to any namespace diff --git a/charts/druid/templates/service-account.yaml b/charts/druid/templates/service-account.yaml index 0a661ada0..a63074f2e 100644 --- a/charts/druid/templates/service-account.yaml +++ b/charts/druid/templates/service-account.yaml @@ -5,4 +5,4 @@ metadata: name: etcd-druid namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid diff --git a/charts/druid/templates/service.yaml b/charts/druid/templates/service.yaml index 82b752505..2386073b7 100644 --- a/charts/druid/templates/service.yaml +++ b/charts/druid/templates/service.yaml @@ -2,20 +2,24 @@ apiVersion: v1 kind: Service metadata: - labels: - gardener.cloud/role: etcd-druid name: etcd-druid namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd-druid spec: type: ClusterIP selector: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid ports: + {{- if (((.Values.controllerManager).server).metrics).port }} - name: metrics - port: 8080 + port: {{ .Values.controllerManager.server.metrics.port }} protocol: TCP - targetPort: 8080 + targetPort: {{ .Values.controllerManager.server.metrics.port }} + {{- end }} + {{- if (((.Values.controllerManager).server).webhook).port }} - name: webhooks - port: 9443 + port: {{ .Values.controllerManager.server.webhook.port }} protocol: TCP - targetPort: 9443 + targetPort: {{ .Values.controllerManager.server.webhook.port }} + {{- end }} diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index a2093bcc0..c9781394a 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -5,8 +5,9 @@ metadata: name: etcd-druid namespace: {{ .Release.Namespace }} labels: - gardener.cloud/role: etcd-druid + app.kubernetes.io/name: etcd-druid webhooks: + {{- if ((.Values.webhooks).sentinel).enabled }} - admissionReviewVersions: - v1beta1 - v1 @@ -16,7 +17,7 @@ webhooks: name: etcd-druid namespace: {{ .Release.Namespace }} path: /webhooks/sentinel - port: 9443 + port: {{ .Values.controllerManager.server.webhook.port }} failurePolicy: Fail matchPolicy: Exact name: sentinel.webhooks.druid.gardener.cloud @@ -89,3 +90,4 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 10 + {{- end }} diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index 76a31ac5c..ff57cb6de 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -1,26 +1,56 @@ -crds: - enabled: true image: repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcd-druid tag: latest imagePullPolicy: IfNotPresent replicas: 1 -ignoreOperationAnnotation: false -# etcdStatusSyncPeriod: 15s -# enableBackupCompaction: true -# eventsThreshold: 15 -# metricsScrapeWaitDuration: "10s" -# featureGates: -# UseEtcdWrapper: true +resources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 50m + memory: 128Mi -server: - webhook: - bindAddress: "" - port: 9443 - tls: - serverCertDir: /etc/webhook-server-tls +featureGates: + UseEtcdWrapper: true -sentinelWebhook: - enabled: true - exemptServiceAccounts: - - system:serviceaccount:kube-system:generic-garbage-collector +controllerManager: + server: + metrics: + bindAddress: "" + port: 8080 + webhook: + bindAddress: "" + port: 9443 + tls: + serverCertDir: /etc/webhook-server-tls + leaderElection: + enabled: true + id: druid-leader-election + disableLeaseCache: true + +controllers: + etcd: + workers: 3 + enableEtcdSpecAutoReconcile: false + disableEtcdServiceAccountAutomount: false + etcdStatusSyncPeriod: 15s + etcdMemberNotReadyThreshold: 5m + etcdMemberUnknownThreshold: 1m + compaction: + enabled: true + workers: 3 + etcdEventsThreshold: 1000000 + activeDeadlineDuration: 3h + metricsScrapeWaitDuration: 0s + etcdCopyBackupsTask: + workers: 3 + secret: + workers: 10 + +webhooks: + sentinel: + enabled: true + # reconciler-service-account: system:serviceaccount:{{ .Release.Namespace }}:etcd-druid + exemptServiceAccounts: + - system:serviceaccount:kube-system:generic-garbage-collector diff --git a/docs/concepts/controllers.md b/docs/concepts/controllers.md index 539255774..c6ad4fec7 100644 --- a/docs/concepts/controllers.md +++ b/docs/concepts/controllers.md @@ -8,7 +8,7 @@ All controllers that are a part of etcd-druid reside in package `internal/contro etcd-druid currently consists of 5 controllers, each having its own responsibility: -- *etcd* : responsible for the reconciliation of the `Etcd` CR spec, which allows users to run etcd clusters within the specified Kubernetes cluster, and also responsible for periodically updating the `Etcd` CR status with the up-to-date state of the managed etcd cluster. +- *etcd* : responsible for the reconciliation of the `Etcd` CR spec, which allows users to run etcd clusters within the specified Kubernetes cluster, and also responsible for periodically updating the `Etcd` CR status with the up-to-date state of the managed etcd cluster. - *compaction* : responsible for [snapshot compaction](/docs/proposals/02-snapshot-compaction.md). - *etcdcopybackupstask* : responsible for the reconciliation of the `EtcdCopyBackupsTask` CR, which helps perform the job of copying snapshot backups from one object store to another. - *secret* : responsible in making sure `Secret`s being referenced by `Etcd` resources are not deleted while in use. @@ -44,13 +44,13 @@ The logic relevant to the controller manager like the creation of the controller The *etcd controller* is responsible for the reconciliation of the `Etcd` resource spec and status. It handles the provisioning and management of the etcd cluster. Different components that are required for the functioning of the cluster like `Leases`, `ConfigMap`s, and the `Statefulset` for the etcd cluster are all deployed and managed by the *etcd controller*. -Additionally, *etcd controller* also periodically updates the `Etcd` resource status with latest available information from the etcd cluster, as well as results and errors from the recentmost reconciliation of the `Etcd` resource spec. +Additionally, *etcd controller* also periodically updates the `Etcd` resource status with the latest available information from the etcd cluster, as well as results and errors from the recent-most reconciliation of the `Etcd` resource spec. The *etcd controller* is essential to the functioning of the etcd cluster and etcd-druid, thus the minimum number of worker threads is 1 (default being 3). ### `Etcd` Spec Reconciliation -While building the controller, an event filter is set such that the behavior of the controller depends on the `gardener.cloud/operation: reconcile` *annotation*. This is controlled by the `--enable-etcd-spec-auto-reconcile` CLI flag, which if set to `false`, tells the controller to perform reconciliation only when this annotation is present. If the flag is set to `true`, the controller will reconcile the etcd cluster anytime the `Etcd` spec, and thus `generation`, changes, and the next queued event for it is triggered. +While building the controller, an event filter is set such that the behavior of the controller depends on the `gardener.cloud/operation: reconcile` *annotation*. This is controlled by the `--enable-etcd-spec-auto-reconcile` CLI flag, which, if set to `false`, tells the controller to perform reconciliation only when this annotation is present. If the flag is set to `true`, the controller will reconcile the etcd cluster anytime the `Etcd` spec, and thus `generation`, changes, and the next queued event for it is triggered. The reason this filter is present is that any disruption in the `Etcd` resource due to reconciliation (due to changes in the `Etcd` spec, for example) while workloads are being run would cause unwanted downtimes to the etcd cluster. Hence, any user who wishes to avoid such disruptions, can choose to set the `--enable-etcd-spec-auto-reconcile` CLI flag to `false`. An example of this is Gardener's [gardenlet](https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md), which reconciles the `Etcd` resource only during a shoot cluster's [*maintenance window*](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_maintenance.md). @@ -63,7 +63,7 @@ The `Etcd` resource status is updated periodically by `etcd controller`, the int Status fields of the `Etcd` resource such as `LastOperation`, `LastErrors` and `ObservedGeneration`, are updated to reflect the result of the recent reconciliation of the `Etcd` resource spec. - `LastOperation` holds information about the last operation performed on the etcd cluster, indicated by fields `Type`, `State`, `Description` and `LastUpdateTime`. Additionally, a field `RunID` indicates the unique ID assigned to the specific reconciliation run, to allow for better debugging of issues. -- `LastErrors` is a slice of errors encountered by the last reconciliation run. Each error consists of fields `Code` to indicate the custom druid error code for the error, a human-readable `Description`, and the `ObservedAt` time when the error was seen. +- `LastErrors` is a slice of errors encountered by the last reconciliation run. Each error consists of fields `Code` to indicate the custom druid error code for the error, a human-readable `Description`, and the `ObservedAt` time when the error was seen. - `ObservedGeneration` indicates the latest `generation` of the `Etcd` resource that druid has "observed" and consequently reconciled. It helps identify whether a change in the `Etcd` resource spec was acted upon by druid or not. Status fields of the `Etcd` resource which correspond to the `StatefulSet` like `CurrentReplicas`, `ReadyReplicas` and `Replicas` are updated to reflect those of the `StatefulSet` by the controller. @@ -75,8 +75,8 @@ Status fields related to the etcd cluster itself, such as `Members`, `PeerUrlTLS `Etcd` resource conditions are indicated by status field `Conditions`. The condition checks that are currently performed are: - `AllMembersReady`: indicates readiness of all members of the etcd cluster. -- `Ready`: indicates overall readiness of the etcd cluster in serving traffic. -- `BackupReady`: indicates health of the etcd backups, ie, whether etcd backups are being taken regularly as per schedule. This condition is applicable only when backups are enabled for the etcd cluster. +- `Ready`: indicates overall readiness of the etcd cluster in serving traffic. +- `BackupReady`: indicates health of the etcd backups, i.e., whether etcd backups are being taken regularly as per schedule. This condition is applicable only when backups are enabled for the etcd cluster. - `DataVolumesReady`: indicates health of the persistent volumes containing the etcd data. ## Compaction Controller diff --git a/docs/concepts/webhooks.md b/docs/concepts/webhooks.md index 3490633df..54df0575f 100644 --- a/docs/concepts/webhooks.md +++ b/docs/concepts/webhooks.md @@ -19,7 +19,7 @@ internal/webhook/sentinel - `register.go`: contains the logic for registering the webhook with the etcd-druid controller manager. - `handler.go`: contains the webhook admission handler logic. -Each webhook package may also contains auxiliary files which are relevant to that specific webhook. +Each webhook package may also contain auxiliary files which are relevant to that specific webhook. ## Sentinel Webhook diff --git a/docs/deployment/cli-flags.md b/docs/deployment/cli-flags.md index e58be11d0..dec476f2f 100644 --- a/docs/deployment/cli-flags.md +++ b/docs/deployment/cli-flags.md @@ -4,7 +4,10 @@ Etcd-druid exposes the following CLI flags that allow for configuring its behavi | CLI FLag | Component | Description | Default | |-----------------------------------------|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------| -| `metrics-addr` | `controller-manager` | The address the metric endpoint binds to. | `":8080"` | +| `feature-gates` | `etcd-druid` | A set of key=value pairs that describe feature gates for alpha/experimental features. Please check [feature-gates](feature-gates.md) for more information. | `""` | +| `metrics-bind-address` | `controller-manager` | The IP address that the metrics endpoint binds to. | `""` | +| `metrics-port` | `controller-manager` | The port used for the metrics endpoint. | `8080` | +| `metrics-addr` | `controller-manager` | The fully qualified address:port that the metrics endpoint binds to.
Deprecated: this field will be eventually removed. Please use `--metrics-bind-address` and --`metrics-port` instead. | `":8080"` | | `webhook-server-bind-address` | `controller-manager` | The IP address on which to listen for the HTTPS webhook server. | `""` | | `webhook-server-port` | `controller-manager` | The port on which to listen for the HTTPS webhook server. | `9443` | | `webhook-server-tls-server-cert-dir` | `controller-manager` | The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively). | `"/etc/webhook-server-tls"` | @@ -27,5 +30,5 @@ Etcd-druid exposes the following CLI flags that allow for configuring its behavi | `etcd-copy-backups-task-workers` | `etcdcopybackupstask-controller` | Number of worker threads for the etcdcopybackupstask controller. | `3` | | `secret-workers` | `secret-controller` | Number of worker threads for the secrets controller. | `10` | | `enable-sentinel-webhook` | `sentinel-webhook` | Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid. | `false` | -| `reconciler-service-account` | `sentinel-webhook` | The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. | `` | +| `reconciler-service-account` | `sentinel-webhook` | The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. If unspecified, the default service account mounted for etcd-druid will be used. | `` | | `sentinel-exempt-service-accounts` | `sentinel-webhook` | The comma-separated list of fully qualified names of service accounts that are exempt from Sentinel Webhook checks. | `""` | diff --git a/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh b/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh index b2bdc8730..54bd0abf6 100644 --- a/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh +++ b/hack/e2e-test/infrastructure/overlays/aws/common/files/common.sh @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors # # SPDX-License-Identifier: Apache-2.0 -set -xe +set -e if [[ -n "${LOCALSTACK_HOST}" ]]; then export AWS_ENDPOINT_URL_S3="http://${LOCALSTACK_HOST}" diff --git a/internal/controller/config.go b/internal/controller/config.go index 8abb96088..486336ae0 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -22,6 +22,8 @@ import ( const ( metricsAddrFlagName = "metrics-addr" + metricsBindAddressFlagName = "metrics-bind-address" + metricsPortFlagName = "metrics-port" webhookServerBindAddressFlagName = "webhook-server-bind-address" webhookServerPortFlagName = "webhook-server-port" webhookServerTLSServerCertDir = "webhook-server-tls-server-cert-dir" @@ -30,7 +32,9 @@ const ( leaderElectionResourceLockFlagName = "leader-election-resource-lock" disableLeaseCacheFlagName = "disable-lease-cache" - defaultMetricsAddr = ":8080" + defaultMetricsAddr = "" + defaultMetricsBindAddress = "" + defaultMetricsPort = 8080 defaultWebhookServerBindAddress = "" defaultWebhookServerPort = 9443 defaultWebhookServerTLSServerCert = "/etc/webhook-server-tls" @@ -114,8 +118,12 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { cfg.Server.Webhook = HTTPSServer{} cfg.Server.Webhook.Server = Server{} + flag.StringVar(&cfg.Server.Metrics.BindAddress, metricsBindAddressFlagName, defaultMetricsBindAddress, + "The IP address that the metrics endpoint binds to.") + flag.IntVar(&cfg.Server.Metrics.Port, metricsPortFlagName, defaultMetricsPort, + "The port used for the metrics endpoint.") flag.StringVar(&cfg.Server.Metrics.BindAddress, metricsAddrFlagName, defaultMetricsAddr, - "The address the metric endpoint binds to.") + fmt.Sprintf("The fully qualified address:port that the metrics endpoint binds to. Deprecated: this field will be eventually removed. Please use %s and %s instead.", metricsBindAddressFlagName, metricsPortFlagName)) flag.StringVar(&cfg.Server.Webhook.Server.BindAddress, webhookServerBindAddressFlagName, defaultWebhookServerBindAddress, "The IP address on which to listen for the HTTPS webhook server.") flag.IntVar(&cfg.Server.Webhook.Server.Port, webhookServerPortFlagName, defaultWebhookServerPort, @@ -127,7 +135,7 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { flag.StringVar(&cfg.LeaderElectionID, leaderElectionIDFlagName, defaultLeaderElectionID, "Name of the resource that leader election will use for holding the leader lock.") flag.StringVar(&cfg.LeaderElectionResourceLock, leaderElectionResourceLockFlagName, defaultLeaderElectionResourceLock, - "Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.") + "Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'. Deprecated: will be removed in the future in favour of using only `leases` as the leader election resource lock for the controller manager.") flag.BoolVar(&cfg.DisableLeaseCache, disableLeaseCacheFlagName, defaultDisableLeaseCache, "Disable cache for lease.coordination.k8s.io resources.") diff --git a/internal/controller/manager.go b/internal/controller/manager.go index c6fc77e70..b41abebd2 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -6,6 +6,9 @@ package controller import ( "context" + "net" + "strconv" + "strings" "time" "github.com/gardener/etcd-druid/internal/client/kubernetes" @@ -62,6 +65,11 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { uncachedObjects = append(uncachedObjects, &coordinationv1.Lease{}, &coordinationv1beta1.Lease{}) } + // TODO: remove this once `--metrics-addr` flag is removed + if !strings.Contains(config.Server.Metrics.BindAddress, ":") { + config.Server.Metrics.BindAddress = net.JoinHostPort(config.Server.Metrics.BindAddress, strconv.Itoa(config.Server.Metrics.Port)) + } + return ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Client: client.Options{ Cache: &client.CacheOptions{ diff --git a/skaffold.yaml b/skaffold.yaml index 5dfa732a7..d5c3b1ab1 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -23,17 +23,20 @@ deploy: helm: {} skipBuildDependencies: true setValues: - enableBackupCompaction: "true" - eventsThreshold: 15 - metricsScrapeWaitDuration: "30s" + controllers: + compaction: + etcdEventsThreshold: 15 + metricsScrapeWaitDuration: 30s profiles: - name: e2e-test activation: - env: "DRUID_E2E_TEST=true" patches: - op: add - path: /deploy/helm/releases/0/setValues/etcdStatusSyncPeriod - value: "5s" + path: /deploy/helm/releases/0/setValues/controllers + value: + etcd: + etcdStatusSyncPeriod: 5s - name: do-not-use-feature-gates activation: - env: "USE_ETCD_DRUID_FEATURE_GATES=false" From 8e8d589a9016f99f97b937ae2436a7625534442a Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sat, 4 May 2024 13:08:59 +0530 Subject: [PATCH 154/235] Update docs, fix formatting in quorum loss recovery doc --- docs/development/getting-started-locally.md | 10 +- ...m-permanent-quorum-loss-in-etcd-cluster.md | 158 +++++++++--------- internal/controller/manager.go | 2 +- 3 files changed, 87 insertions(+), 83 deletions(-) diff --git a/docs/development/getting-started-locally.md b/docs/development/getting-started-locally.md index 2bc068292..4b441d38f 100644 --- a/docs/development/getting-started-locally.md +++ b/docs/development/getting-started-locally.md @@ -15,9 +15,9 @@ cd etcd-druid > **Note:** > ->- Etcd-druid uses [kind](https://kind.sigs.k8s.io/) as it's local Kubernetes engine. The local setup is configured for kind due to it's convenience but any other kubernetes setup would also work. ->- To setup Etcd-Druid with backups enabled on a [LocalStack](https://github.com/localstack/localstack) provider, refer [this document](getting-started-locally-localstack.md) ->- In the section [Annotate Etcd CR with the reconcile annotation](#annotate-etcd-cr-with-the-reconcile-annotation), the flag `ignore-operation-annotation` is set to false, which means a special annotation is required on the Etcd CR, for etcd-druid to reconcile it. To disable this behavior and allow auto-reconciliation of the Etcd CR for any change in the Etcd spec, set the `ignoreOperationAnnotation` flag to `true` in the `values.yaml` located at [`charts/druid/values.yaml`](../../charts/druid/values.yaml). Or if etcd-druid is being run as a process, then while starting the process, set the CLI flag `--ignore-operation-annotation=true` for it. +>- Etcd-druid uses [kind](https://kind.sigs.k8s.io/) as it's local Kubernetes engine. The local setup is configured for kind due to its convenience but any other kubernetes setup would also work. +>- To set up etcd-druid with backups enabled on a [LocalStack](https://github.com/localstack/localstack) provider, refer [this document](getting-started-locally-localstack.md) +>- In the section [Annotate Etcd CR with the reconcile annotation](#annotate-etcd-cr-with-the-reconcile-annotation), the flag `--enable-etcd-spec-auto-reconcile` is set to `false`, which means a special annotation is required on the Etcd CR, for etcd-druid to reconcile it. To disable this behavior and allow auto-reconciliation of the Etcd CR for any change in the Etcd spec, set the `controllers.etcd.enableEtcdSpecAutoReconcile` value to `true` in the `values.yaml` located at [`charts/druid/values.yaml`](../../charts/druid/values.yaml). Or if etcd-druid is being run as a process, then while starting the process, set the CLI flag `--enable-etcd-spec-auto-reconcile=true` for it. ## Setting up the kind cluster @@ -104,9 +104,9 @@ kubectl apply -f config/samples/druid_v1alpha1_etcd.yaml ### Annotate Etcd CR with the reconcile annotation -> **Note :** If the `ignore-operation-annotation` flag is set to `true`, this step is not required +> **Note :** If the `--enable-etcd-spec-auto-reconcile` flag is set to `true`, this step is not required. -The above step creates an Etcd resource, however etcd-druid won't pick it up for reconciliation without an annotation. To get etcd-druid to reconcile the etcd CR, annotate it with the following `gardener.cloud/operation:reconcile`. +The above step creates an Etcd resource, however etcd-druid won't pick it up for reconciliation without an annotation. To get etcd-druid to reconcile the etcd CR, annotate it with the following `gardener.cloud/operation: reconcile`. ```sh # Annotate etcd-test CR to reconcile diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md index fa73c4a20..ed18b305f 100644 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -1,62 +1,64 @@ # Recovery from Permanent Quorum Loss in an Etcd Cluster ## Quorum loss in Etcd Cluster -[Quorum loss](https://etcd.io/docs/v3.4/op-guide/recovery/) means when majority of Etcd pods(greater than or equal to n/2 + 1) are down simultaneously for some reason. +[Quorum loss](https://etcd.io/docs/v3.4/op-guide/recovery/) means when the majority of Etcd pods(greater than or equal to n/2 + 1) are down simultaneously for some reason. There are two types of quorum loss that can happen to [Etcd multinode cluster](https://github.com/gardener/etcd-druid/tree/master/docs/proposals/multi-node) : -1. **Transient quorum loss** - A quorum loss is called transient when majority of Etcd pods are down simultaneously for some time. The pods may be down due to network unavailability, high resource usages etc. When the pods come back after some time, they can re-join to the cluster and the quorum is recovered automatically without any manual intervention. There should not be a permanent failure for majority of etcd pods due to hardware failure or disk corruption. +1. **Transient quorum loss** - A quorum loss is called transient when the majority of Etcd pods are down simultaneously for some time. The pods may be down due to network unavailability, high resource usages etc. When the pods come back after some time, they can re-join the cluster and quorum is recovered automatically without any manual intervention. There should not be a permanent failure for the majority of etcd pods due to hardware failure or disk corruption. -2. **Permanent quorum loss** - A quorum loss is called permanent when majority of Etcd cluster members experience permanent failure, whether due to hardware failure or disk corruption etc. then the etcd cluster is not going to recover automatically from the quorum loss. A human operator will now need to intervene and execute the following steps to recover the multi-node Etcd cluster. +2. **Permanent quorum loss** - A quorum loss is called permanent when the majority of Etcd cluster members experience permanent failure, whether due to hardware failure or disk corruption etc. then the etcd cluster is not going to recover automatically from the quorum loss. A human operator will now need to intervene and execute the following steps to recover the multi-node Etcd cluster. -If permanent quorum loss occurs to a multinode Etcd cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs etc related to that Etcd cluster and work on those resources only. Following steps guide a human operator to recover from permanent quorum loss of a Etcd cluster. We assume the name of the Etcd CR for the Etcd cluster is `etcd-main`. +If permanent quorum loss occurs to a multinode Etcd cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs, etc. related to that Etcd cluster and work on those resources only. Following steps guide a human operator to recover from permanent quorum loss of an etcd cluster. We assume the name of the Etcd CR for the Etcd cluster is `etcd-main`. **Etcd cluster in shoot control plane of gardener deployment:** -There are two [Etcd clusters](https://github.com/gardener/etcd-druid/tree/master/docs/proposals/multi-node) running in shoot control plane. One is named as `etcd-events` and another is named `etcd-main`. The operator needs to take care of permanent quorum loss to a specific cluster. If permanent quorum loss occurs to `etcd-events` cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs etc related to `etcd-events` cluster and work on those resources only. +There are two [Etcd clusters](https://github.com/gardener/etcd-druid/tree/master/docs/proposals/multi-node) running in shoot control plane. One is named as `etcd-events` and another is named `etcd-main`. The operator needs to take care of permanent quorum loss to a specific cluster. If permanent quorum loss occurs to `etcd-events` cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs, etc. related to `etcd-events` cluster and work on those resources only. :warning: **Note:** Please note that manually restoring etcd can result in data loss. This guide is the last resort to bring an Etcd cluster up and running again. - If etcd-druid and etcd-backup-restore is being used with gardener, then +If etcd-druid and etcd-backup-restore is being used with gardener, then - Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. +Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. - 1. Add the following annotations to the `Etcd` resource `etcd-main`: - 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile="true"` +1. Add the following annotations to the `Etcd` resource `etcd-main`: + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile="true"` - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection="false"` + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection="false"` - 2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: +2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: - ``` - Volumes: + ``` + Volumes: etcd-config-file: - Type: ConfigMap (a volume populated by a ConfigMap) - Name: etcd-bootstrap-4785b0 - Optional: false - ``` + Type: ConfigMap (a volume populated by a ConfigMap) + Name: etcd-bootstrap-4785b0 + Optional: false + ``` - Alternatively, the related configmap name can be obtained by executing following command as well: - `kubectl get sts etcd-main -o jsonpath='{.spec.template.spec.volumes[?(@.name=="etcd-config-file")].configMap.name}'` + Alternatively, the related configmap name can be obtained by executing following command as well: - 3. Scale down the `etcd-main` statefulset replicas to `0` + `kubectl get sts etcd-main -o jsonpath='{.spec.template.spec.volumes[?(@.name=="etcd-config-file")].configMap.name}'` - `kubectl scale sts etcd-main --replicas=0` - 4. The PVCs will look like the following on listing them with the command `kubectl get pvc` : +3. Scale down the `etcd-main` statefulset replicas to `0`: - ``` - main-etcd-etcd-main-0 Bound pv-shoot--garden--aws-ha-dcb51848-49fa-4501-b2f2-f8d8f1fad111 80Gi RWO gardener.cloud-fast 13d - main-etcd-etcd-main-1 Bound pv-shoot--garden--aws-ha-b4751b28-c06e-41b7-b08c-6486e03090dd 80Gi RWO gardener.cloud-fast 13d - main-etcd-etcd-main-2 Bound pv-shoot--garden--aws-ha-ff17323b-d62e-4d5e-a742-9de823621490 80Gi RWO gardener.cloud-fast 13d - ``` - Delete all PVCs that are attached to `etcd-main` cluster. + `kubectl scale sts etcd-main --replicas=0` + +4. The PVCs will look like the following on listing them with the command `kubectl get pvc` : + + ``` + main-etcd-etcd-main-0 Bound pv-shoot--garden--aws-ha-dcb51848-49fa-4501-b2f2-f8d8f1fad111 80Gi RWO gardener.cloud-fast 13d + main-etcd-etcd-main-1 Bound pv-shoot--garden--aws-ha-b4751b28-c06e-41b7-b08c-6486e03090dd 80Gi RWO gardener.cloud-fast 13d + main-etcd-etcd-main-2 Bound pv-shoot--garden--aws-ha-ff17323b-d62e-4d5e-a742-9de823621490 80Gi RWO gardener.cloud-fast 13d + ``` + Delete all PVCs that are attached to `etcd-main` cluster. - `kubectl delete pvc -l instance=etcd-main` + `kubectl delete pvc -l instance=etcd-main` - 5. Check the etcd's member leases. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. - One of those leases will have holder identity as `:Leader` and rest of etcd member leases have holder identities as `:Member`. - Please ignore the snapshot leases i.e those leases which have suffix `snap`. +5. Check the etcd's member leases. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. + One of those leases will have holder identity as `:Leader` and rest of etcd member leases have holder identities as `:Member`. + Please ignore the snapshot leases i.e. those leases which have suffix `snap`. - etcd-main member leases: + etcd-main member leases: ``` NAME HOLDER AGE etcd-main-0 4c37667312a3912b:Member 1m @@ -64,63 +66,65 @@ There are two [Etcd clusters](https://github.com/gardener/etcd-druid/tree/master etcd-main-2 c62ee6af755e890d:Leader 1m ``` - Delete all `etcd-main` member leases. + Delete all `etcd-main` member leases. - 6. Edit the `etcd-main` cluster's configmap (ex: `etcd-bootstrap-4785b0`) as follows: +6. Edit the `etcd-main` cluster's configmap (ex: `etcd-bootstrap-4785b0`) as follows: - Find the `initial-cluster` field in the configmap. It will look like the following: - ``` - # Initial cluster - initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380,etcd-main-1=https://etcd-main-1.etcd-main-peer.default.svc:2380,etcd-main-2=https://etcd-main-2.etcd-main-peer.default.svc:2380 - ``` + Find the `initial-cluster` field in the configmap. It will look like the following: + ``` + # Initial cluster + initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380,etcd-main-1=https://etcd-main-1.etcd-main-peer.default.svc:2380,etcd-main-2=https://etcd-main-2.etcd-main-peer.default.svc:2380 + ``` - Change the `initial-cluster` field to have only one member (`etcd-main-0`) in the string. It should now look like this: + Change the `initial-cluster` field to have only one member (`etcd-main-0`) in the string. It should now look like this: - ``` - # Initial cluster - initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380 - ``` + ``` + # Initial cluster + initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380 + ``` - 7. Scale up the `etcd-main` statefulset replicas to `1` +7. Scale up the `etcd-main` statefulset replicas to `1` - `kubectl scale sts etcd-main --replicas=1` + `kubectl scale sts etcd-main --replicas=1` - 8. Wait for the single-member etcd cluster to be completely ready. +8. Wait for the single-member etcd cluster to be completely ready. - `kubectl get pods etcd-main-0` will give the following output when ready: - ``` - NAME READY STATUS RESTARTS AGE - etcd-main-0 2/2 Running 0 1m - ``` + `kubectl get pods etcd-main-0` will give the following output when ready: + ``` + NAME READY STATUS RESTARTS AGE + etcd-main-0 2/2 Running 0 1m + ``` - 9. Remove the following annotations from the `Etcd` resource `etcd-main`: +9. Remove the following annotations from the `Etcd` resource `etcd-main`: - 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile-` + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile-` - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection-` + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection-` - 10. Finally add the following annotation to the `Etcd` resource `etcd-main`: `kubectl annotate etcd etcd-main gardener.cloud/operation="reconcile"` +10. Finally, add the following annotation to the `Etcd` resource `etcd-main`: - 11. Verify that the etcd cluster is formed correctly. + `kubectl annotate etcd etcd-main gardener.cloud/operation="reconcile"` - All the `etcd-main` pods will have outputs similar to following: - ``` - NAME READY STATUS RESTARTS AGE - etcd-main-0 2/2 Running 0 5m - etcd-main-1 2/2 Running 0 1m - etcd-main-2 2/2 Running 0 1m - ``` - Additionally, check if the Etcd CR is ready with `kubectl get etcd etcd-main` : - ``` - NAME READY AGE - etcd-main true 13d - ``` +11. Verify that the etcd cluster is formed correctly. - Additionally, check the leases for 30 seconds at least. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. One of those leases will have holder identity as `:Leader` and rest of those leases have holder identities as `:Member`. The `AGE` of those leases can also be inspected to identify if those leases were updated in conjunction with the restart of the Etcd cluster: Example: + All the `etcd-main` pods will have outputs similar to following: + ``` + NAME READY STATUS RESTARTS AGE + etcd-main-0 2/2 Running 0 5m + etcd-main-1 2/2 Running 0 1m + etcd-main-2 2/2 Running 0 1m + ``` + Additionally, check if the Etcd CR is ready with `kubectl get etcd etcd-main` : + ``` + NAME READY AGE + etcd-main true 13d + ``` - ``` - NAME HOLDER AGE - etcd-main-0 4c37667312a3912b:Member 1m - etcd-main-1 75a9b74cfd3077cc:Member 1m - etcd-main-2 c62ee6af755e890d:Leader 1m - ``` \ No newline at end of file + Additionally, check the leases for 30 seconds at least. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. One of those leases will have holder identity as `:Leader` and rest of those leases have holder identities as `:Member`. The `AGE` of those leases can also be inspected to identify if those leases were updated in conjunction with the restart of the Etcd cluster: Example: + + ``` + NAME HOLDER AGE + etcd-main-0 4c37667312a3912b:Member 1m + etcd-main-1 75a9b74cfd3077cc:Member 1m + etcd-main-2 c62ee6af755e890d:Leader 1m + ``` diff --git a/internal/controller/manager.go b/internal/controller/manager.go index b41abebd2..8fb2fada9 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -65,7 +65,7 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { uncachedObjects = append(uncachedObjects, &coordinationv1.Lease{}, &coordinationv1beta1.Lease{}) } - // TODO: remove this once `--metrics-addr` flag is removed + // TODO: remove this check once `--metrics-addr` flag is removed, and directly compute the address:port when setting managerOptions.Metrics.BindAddress if !strings.Contains(config.Server.Metrics.BindAddress, ":") { config.Server.Metrics.BindAddress = net.JoinHostPort(config.Server.Metrics.BindAddress, strconv.Itoa(config.Server.Metrics.Port)) } From 2acca086b278ccd12b00cc81999a066657dbf09d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 6 May 2024 11:06:41 +0530 Subject: [PATCH 155/235] addressed review comments --- ...m-permanent-quorum-loss-in-etcd-cluster.md | 6 ++--- internal/common/constants.go | 8 +++---- internal/controller/compaction/reconciler.go | 4 ++-- internal/controller/config.go | 24 +++++++++---------- internal/controller/etcd/reconcile_delete.go | 1 + internal/controller/etcd/reconcile_spec.go | 3 +++ internal/controller/etcd/register.go | 20 +++++++--------- .../etcdcopybackupstask/reconciler.go | 24 +++++++++---------- .../etcdcopybackupstask/reconciler_test.go | 10 ++++---- internal/controller/manager.go | 6 ++--- internal/operator/statefulset/builder.go | 4 ++-- internal/operator/statefulset/stsmatcher.go | 4 ++-- internal/utils/envvar.go | 12 +++++----- .../controllers/compaction/reconciler_test.go | 10 ++++---- 14 files changed, 68 insertions(+), 68 deletions(-) diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md index ed18b305f..b12b0f112 100644 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -21,9 +21,9 @@ If etcd-druid and etcd-backup-restore is being used with gardener, then Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. 1. Add the following annotations to the `Etcd` resource `etcd-main`: - 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile="true"` + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile='true'` - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection="false"` + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection='false'` 2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: @@ -103,7 +103,7 @@ Target the control plane of affected shoot cluster via `kubectl`. Alternatively, 10. Finally, add the following annotation to the `Etcd` resource `etcd-main`: - `kubectl annotate etcd etcd-main gardener.cloud/operation="reconcile"` + `kubectl annotate etcd etcd-main gardener.cloud/operation='reconcile'` 11. Verify that the etcd cluster is formed correctly. diff --git a/internal/common/constants.go b/internal/common/constants.go index f841a963f..9d51e2872 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -150,10 +150,10 @@ const ( // BackupRestoreClientTLSVolumeMountPath is the path on a container where the client certificate-key pair used by the client to communicate to the backup-restore server is mounted. BackupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" - // GCSBackupVolumeMountPath is the path on a container where the GCS backup secret is mounted. - GCSBackupVolumeMountPath = "/var/.gcp/" - // NonGCSProviderBackupVolumeMountPath is the path on a container where the non-GCS provider backup secret is mounted. - NonGCSProviderBackupVolumeMountPath = "/var/etcd-backup" + // GCSBackupSecretVolumeMountPath is the path on a container where the GCS backup secret is mounted. + GCSBackupSecretVolumeMountPath = "/var/.gcp/" + // NonGCSProviderBackupSecretVolumeMountPath is the path on a container where the non-GCS provider backup secret is mounted. + NonGCSProviderBackupSecretVolumeMountPath = "/var/etcd-backup" // EtcdDataVolumeMountPath is the path on a container where the etcd data directory is hosted. EtcdDataVolumeMountPath = "/var/etcd/data" diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 35a34d6fe..d35d242e5 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -380,12 +380,12 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu case utils.GCS: vms = append(vms, v1.VolumeMount{ Name: common.ProviderBackupSecretVolumeName, - MountPath: common.GCSBackupVolumeMountPath, + MountPath: common.GCSBackupSecretVolumeMountPath, }) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: vms = append(vms, v1.VolumeMount{ Name: common.ProviderBackupSecretVolumeName, - MountPath: common.NonGCSProviderBackupVolumeMountPath, + MountPath: common.NonGCSProviderBackupSecretVolumeMountPath, }) } diff --git a/internal/controller/config.go b/internal/controller/config.go index 486336ae0..ed20be5dd 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -77,14 +77,14 @@ type Server struct { // LeaderElectionConfig defines the configuration for the leader election for the controller manager. type LeaderElectionConfig struct { - // EnableLeaderElection specifies whether to enable leader election for controller manager. - EnableLeaderElection bool - // LeaderElectionID is the name of the resource that leader election will use for holding the leader lock. - LeaderElectionID string - // LeaderElectionResourceLock specifies which resource type to use for leader election. + // Enabled specifies whether to enable leader election for controller manager. + Enabled bool + // ID is the name of the resource that leader election will use for holding the leader lock. + ID string + // ResourceLock specifies which resource type to use for leader election. // Deprecated: K8S Leases will be used for leader election. No other resource type would be permitted. // This configuration option will be removed eventually. It is advisable to not use this option any longer. - LeaderElectionResourceLock string + ResourceLock string } // ManagerConfig defines the configuration for the controller manager. @@ -93,8 +93,8 @@ type ManagerConfig struct { // Deprecated: This field will be eventually removed. Please use Server.Metrics.BindAddress instead. MetricsAddr string // Server is the configuration for the HTTP server. - Server *ServerConfig - LeaderElectionConfig + Server *ServerConfig + LeaderElection LeaderElectionConfig // DisableLeaseCache specifies whether to disable cache for lease.coordination.k8s.io resources. DisableLeaseCache bool // FeatureGates contains the feature gates to be used by etcd-druid. @@ -130,11 +130,11 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { "The port on which to listen for the HTTPS webhook server.") flag.StringVar(&cfg.Server.Webhook.TLS.ServerCertDir, webhookServerTLSServerCertDir, defaultWebhookServerTLSServerCert, "The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively).") - flag.BoolVar(&cfg.EnableLeaderElection, enableLeaderElectionFlagName, defaultEnableLeaderElection, + flag.BoolVar(&cfg.LeaderElection.Enabled, enableLeaderElectionFlagName, defaultEnableLeaderElection, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&cfg.LeaderElectionID, leaderElectionIDFlagName, defaultLeaderElectionID, + flag.StringVar(&cfg.LeaderElection.ID, leaderElectionIDFlagName, defaultLeaderElectionID, "Name of the resource that leader election will use for holding the leader lock.") - flag.StringVar(&cfg.LeaderElectionResourceLock, leaderElectionResourceLockFlagName, defaultLeaderElectionResourceLock, + flag.StringVar(&cfg.LeaderElection.ResourceLock, leaderElectionResourceLockFlagName, defaultLeaderElectionResourceLock, "Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'. Deprecated: will be removed in the future in favour of using only `leases` as the leader election resource lock for the controller manager.") flag.BoolVar(&cfg.DisableLeaseCache, disableLeaseCacheFlagName, defaultDisableLeaseCache, "Disable cache for lease.coordination.k8s.io resources.") @@ -190,7 +190,7 @@ func (cfg *ManagerConfig) populateControllersFeatureGates() { // Validate validates the controller manager config. func (cfg *ManagerConfig) Validate() error { - if err := utils.ShouldBeOneOfAllowedValues("LeaderElectionResourceLock", getAllowedLeaderElectionResourceLocks(), cfg.LeaderElectionResourceLock); err != nil { + if err := utils.ShouldBeOneOfAllowedValues("ResourceLock", getAllowedLeaderElectionResourceLocks(), cfg.LeaderElection.ResourceLock); err != nil { return err } diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index e179ba6b7..1b4462911 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -42,6 +42,7 @@ func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjK operators := r.operatorRegistry.AllOperators() deleteTasks := make([]utils.OperatorTask, 0, len(operators)) for kind, operator := range operators { + // TODO: once we move to go 1.22 (https://go.dev/blog/loopvar-preview) operator := operator deleteTasks = append(deleteTasks, utils.OperatorTask{ Name: fmt.Sprintf("triggerDeletionFlow-%s-operator", kind), diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 3f2c4217e..c6f014af5 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -42,6 +42,7 @@ func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, et withOpAnnotation := etcd.DeepCopy() delete(etcd.Annotations, v1beta1constants.GardenerOperation) if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { + ctx.Logger.Error(err, "failed to remove operation annotation") return ctrlutils.ReconcileWithError(err) } } @@ -56,6 +57,7 @@ func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey for _, kind := range resourceOperators { op := r.operatorRegistry.GetOperator(kind) if err := op.Sync(ctx, etcd); err != nil { + ctx.Logger.Error(err, "failed to sync etcd resource", "kind", kind) return ctrlutils.ReconcileWithError(err) } } @@ -70,6 +72,7 @@ func (r *Reconciler) updateObservedGeneration(ctx component.OperatorContext, etc originalEtcd := etcd.DeepCopy() etcd.Status.ObservedGeneration = &etcd.Generation if err := r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)); err != nil { + ctx.Logger.Error(err, "failed to patch status.ObservedGeneration") return ctrlutils.ReconcileWithError(err) } ctx.Logger.Info("patched status.ObservedGeneration", "ObservedGeneration", etcd.Generation) diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 2e1a1c5ff..2913639a8 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -34,11 +34,11 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { return builder.Complete(r) } -// buildPredicate returns a predicate that filters out events that are not relevant for the Etcd controller. +// buildPredicate returns a predicate that filters events that are relevant for the Etcd controller. // NOTE: // For all conditions the following is applicable: -// 1. create and delete events are always reconciled. -// 2. generic events are not reconciled. If there is a need in future to react to generic events then this should be changed. +// 1. create and delete events are always reconciled irrespective of whether reconcile annotation is present or auto-reconcile has been enabled. +// 2. generic events are never reconciled. If there is a need in future to react to generic events then this should be changed. // Conditions for reconciliation: // Scenario 1: {Auto-Reconcile: false, Reconcile-Annotation-Present: false, Spec-Updated: false/true, Status-Updated: false/true, update-event-reconciled: false} // Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} @@ -55,19 +55,15 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { // Scenario 13: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} // Scenario 14: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} func (r *Reconciler) buildPredicate() predicate.Predicate { - /* - If there is no change to spec and status then no reconciliation would happen. This is also true when auto-reconcile - has been enabled. If an operator wishes to force a reconcile especially when no change (spec/status) has been done to the etcd resource - then the only way is to explicitly add the reconcile annotation to the etcd resource. - */ + // If there is no change to spec and status then no reconciliation would happen. This is also true when auto-reconcile + // has been enabled. If an operator wishes to force a reconcile especially when no change (spec/status) has been done to the etcd resource + // then the only way is to explicitly add the reconcile annotation to the etcd resource. forceReconcilePredicate := predicate.And( r.hasReconcileAnnotation(), noSpecAndStatusUpdated(), ) - /* - If there is a spec change (irrespective of status change) and if there is an update event then it will trigger a reconcile only when either - auto-reconcile has been enabled or an operator has added the reconcile annotation to the etcd resource. - */ + // If there is a spec change (irrespective of status change) and if there is an update event then it will trigger a reconcile only when either + // auto-reconcile has been enabled or an operator has added the reconcile annotation to the etcd resource. onSpecChangePredicate := predicate.And( predicate.Or( r.hasReconcileAnnotation(), diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index cbe256143..518692fb1 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -493,26 +493,26 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum case utils.GCS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, - MountPath: getGCSVolumeMountPathWithPrefixAndSuffix(getVolumeNamePrefix(volumeMountPrefix), "/"), + MountPath: getGCSSecretVolumeMountPathWithPrefixAndSuffix(getVolumeNamePrefix(volumeMountPrefix), "/"), }) case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, - MountPath: getNonGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), + MountPath: getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), }) } return } -func getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { +func getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { // "/var/etcd-backup" - tokens := strings.Split(strings.Trim(common.NonGCSProviderBackupVolumeMountPath, "/"), "/") + tokens := strings.Split(strings.Trim(common.NonGCSProviderBackupSecretVolumeMountPath, "/"), "/") return fmt.Sprintf("/%s/%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) } -func getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { +func getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { // "/var/.gcp" - tokens := strings.Split(strings.TrimSuffix(common.GCSBackupVolumeMountPath, "/"), ".") + tokens := strings.Split(strings.TrimSuffix(common.GCSBackupSecretVolumeMountPath, "/"), ".") return fmt.Sprintf("%s.%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) } @@ -524,17 +524,17 @@ func createEnvVarsFromStore(store *druidv1alpha1.StoreSpec, storeProvider, envKe envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvStorageContainer, *store.Container)) switch storeProvider { case utils.S3: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.ABS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.GCS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"))) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"))) case utils.Swift: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.OCS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) case utils.OSS: - envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) + envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) } return envVars } diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 7eeb623e9..2dd8861b9 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -472,10 +472,10 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { expectedMountPath = *storeSpec.Container case utils.GCS: expectedMountName = volumeMountPrefix + common.ProviderBackupSecretVolumeName - expectedMountPath = getGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") + expectedMountPath = getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: expectedMountName = volumeMountPrefix + common.ProviderBackupSecretVolumeName - expectedMountPath = getNonGCSVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") + expectedMountPath = getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") default: Fail(fmt.Sprintf("Unknown provider: %s", provider)) } @@ -707,12 +707,12 @@ func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefi case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], - Value: getNonGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""), + Value: getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""), }) case utils.GCS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], - Value: getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"), + Value: getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"), }) } Expect(envVars).To(Equal(expected)) @@ -898,7 +898,7 @@ func getProviderEnvElements(storeProvider, prefix, volumePrefix string) Elements return Elements{ prefix + common.EnvGoogleApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(prefix + common.EnvGoogleApplicationCredentials), - "Value": Equal(getGCSVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json")), + "Value": Equal(getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json")), }), } case "Swift": diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 8fb2fada9..86fefc9b2 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -85,9 +85,9 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { Port: config.Server.Webhook.Port, CertDir: config.Server.Webhook.TLS.ServerCertDir, }), - LeaderElection: config.EnableLeaderElection, - LeaderElectionID: config.LeaderElectionID, - LeaderElectionResourceLock: config.LeaderElectionResourceLock, + LeaderElection: config.LeaderElection.Enabled, + LeaderElectionID: config.LeaderElection.ID, + LeaderElectionResourceLock: config.LeaderElection.ResourceLock, }) } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index bdc36b168..2dd53ba3a 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -324,12 +324,12 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { case utils.GCS: return &corev1.VolumeMount{ Name: common.ProviderBackupSecretVolumeName, - MountPath: common.GCSBackupVolumeMountPath, + MountPath: common.GCSBackupSecretVolumeMountPath, } case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: return &corev1.VolumeMount{ Name: common.ProviderBackupSecretVolumeName, - MountPath: common.NonGCSProviderBackupVolumeMountPath, + MountPath: common.NonGCSProviderBackupSecretVolumeMountPath, } } return nil diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 926263673..bdeef5d29 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -372,9 +372,9 @@ func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.Gomega } } case utils.GCS: - return matchVolMount(common.ProviderBackupSecretVolumeName, common.GCSBackupVolumeMountPath) + return matchVolMount(common.ProviderBackupSecretVolumeName, common.GCSBackupSecretVolumeMountPath) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - return matchVolMount(common.ProviderBackupSecretVolumeName, common.NonGCSProviderBackupVolumeMountPath) + return matchVolMount(common.ProviderBackupSecretVolumeName, common.NonGCSProviderBackupSecretVolumeMountPath) } return nil } diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index d1b0e46e5..935c651db 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -61,20 +61,20 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) switch provider { case S3: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) case ABS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) case GCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.GCSBackupVolumeMountPath))) + envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.GCSBackupSecretVolumeMountPath))) envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) case Swift: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) case OSS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) case ECS: if store.SecretRef == nil { @@ -85,7 +85,7 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) case OCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.NonGCSProviderBackupVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) } return envVars, nil diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index f8f0dad36..5ee6164d3 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -391,7 +391,7 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J "VolumeMounts": MatchElements(testutils.VolumeMountIterator, IgnoreExtras, Elements{ common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.ProviderBackupSecretVolumeName), - "MountPath": Equal(common.GCSBackupVolumeMountPath), + "MountPath": Equal(common.GCSBackupSecretVolumeMountPath), }), }), "Env": MatchAllElements(testutils.EnvIterator, Elements{ @@ -485,7 +485,7 @@ func validateStoreAWSForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J }), common.EnvAWSApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAWSApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), + "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), }), }), }), @@ -541,7 +541,7 @@ func validateStoreAzureForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1 }), common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAzureApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), + "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), }), }), }), @@ -597,7 +597,7 @@ func validateStoreOpenstackForCompactionJob(instance *druidv1alpha1.Etcd, j *bat }), common.EnvOpenstackApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvOpenstackApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), + "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), }), }), }), @@ -654,7 +654,7 @@ func validateStoreAlicloudForCompactionJob(instance *druidv1alpha1.Etcd, j *batc }), common.EnvAlicloudApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAlicloudApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupVolumeMountPath), + "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), }), }), }), From d486f678f563fe7e5dbe0e72d1cbbfd63aebaaab Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 7 May 2024 13:00:03 +0530 Subject: [PATCH 156/235] disable-resource-protection annotation will no longer have a value --- api/v1alpha1/types_constant.go | 4 ++-- api/v1alpha1/types_etcd.go | 9 +++------ api/v1alpha1/types_etcd_test.go | 11 +++-------- internal/webhook/sentinel/handler.go | 2 +- internal/webhook/sentinel/handler_test.go | 12 ++++++------ 5 files changed, 15 insertions(+), 23 deletions(-) diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/types_constant.go index 6c3d37d41..ae09297da 100644 --- a/api/v1alpha1/types_constant.go +++ b/api/v1alpha1/types_constant.go @@ -25,6 +25,6 @@ const ( IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" // SuspendEtcdSpecReconcileAnnotation is an annotation set by an operator to temporarily suspend any etcd spec reconciliation. SuspendEtcdSpecReconcileAnnotation = "druid.gardener.cloud/suspend-etcd-spec-reconcile" - // ResourceProtectionAnnotation is an annotation set by an operator to enable or disable protection of resources created by etcd-druid. - ResourceProtectionAnnotation = "druid.gardener.cloud/resource-protection" + // DisableResourceProtectionAnnotation is an annotation set by an operator to disable protection of resources managed by etcd-druid. + DisableResourceProtectionAnnotation = "druid.gardener.cloud/disable-resource-protection" ) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 51200dec0..1cd547105 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -597,13 +597,10 @@ func (e *Etcd) IsReconciliationSuspended() bool { return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) } -// AreManagedResourcesProtected returns true if the Etcd resource has the resource protection annotation set to true, -// else returns false. +// AreManagedResourcesProtected returns true if the Etcd resource has the disable-resource-protection annotation set, +// else returns false func (e *Etcd) AreManagedResourcesProtected() bool { - if metav1.HasAnnotation(e.ObjectMeta, ResourceProtectionAnnotation) { - return e.GetAnnotations()[ResourceProtectionAnnotation] != "false" - } - return true + return !metav1.HasAnnotation(e.ObjectMeta, DisableResourceProtectionAnnotation) } // IsReconciliationInProgress returns true if the Etcd resource is currently being reconciled, else returns false. diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index afa11f07e..8f0c6ecbb 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -211,20 +211,15 @@ var _ = Describe("Etcd", func() { }) Context("AreManagedResourcesProtected", func() { - Context("when etcd has annotation druid.gardener.cloud/resource-protection: false", func() { + Context("when etcd has annotation druid.gardener.cloud/disable-resource-protection", func() { It("should return false", func() { etcd.Annotations = map[string]string{ - ResourceProtectionAnnotation: "false", + DisableResourceProtectionAnnotation: "", } Expect(etcd.AreManagedResourcesProtected()).To(Equal(false)) }) }) - Context("when etcd has annotation druid.gardener.cloud/resource-protection: true", func() { - It("should return true", func() { - Expect(etcd.AreManagedResourcesProtected()).To(Equal(true)) - }) - }) - Context("when etcd does not have annotation druid.gardener.cloud/resource-protection set", func() { + Context("when etcd does not have annotation druid.gardener.cloud/disable-resource-protection set", func() { It("should return true", func() { Expect(etcd.AreManagedResourcesProtected()).To(Equal(true)) }) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 14d4769b6..fed4d2a1a 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -90,7 +90,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R // allow changes to resources if etcd has annotation druid.gardener.cloud/resource-protection: false if !etcd.AreManagedResourcesProtected() { - return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", etcd.Name, druidv1alpha1.ResourceProtectionAnnotation)) + return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } // allow operations on resources if any etcd operation is currently being reconciled, but only by etcd-druid, diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index c2b5f49ab..1084489a3 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -202,11 +202,11 @@ func TestHandleUpdate(t *testing.T) { expectedCode int32 }{ { - name: "resource protection annotation set to false", + name: "disable resource protection annotation set", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, + etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, expectedAllowed: true, - expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { @@ -461,11 +461,11 @@ func TestHandleDelete(t *testing.T) { expectedCode int32 }{ { - name: "resource protection annotation set to false", + name: "disable resource protection annotation set", objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdAnnotations: map[string]string{druidv1alpha1.ResourceProtectionAnnotation: "false"}, + etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, expectedAllowed: true, - expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s: false", testEtcdName, druidv1alpha1.ResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { From 76875635c2e3d6c3f4fb8752f8fcdf207cdd53b6 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 7 May 2024 13:03:35 +0530 Subject: [PATCH 157/235] Run `make generate` --- .../10-crd-druid.gardener.cloud_etcds.yaml | 13 +++++++------ .../bases/10-crd-druid.gardener.cloud_etcds.yaml | 13 +++++++------ docs/concepts/webhooks.md | 2 +- ...ry-from-permanent-quorum-loss-in-etcd-cluster.md | 4 ++-- internal/webhook/sentinel/handler.go | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index 6a66b2b81..b09430040 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -339,7 +339,8 @@ spec: type: object type: object etcd: - description: EtcdConfig defines parameters associated etcd deployed + description: EtcdConfig defines the configuration for the etcd cluster + to be deployed. properties: authSecretRef: description: SecretReference represents a Secret Reference. It @@ -1694,7 +1695,7 @@ spec: type: string autoCompactionRetention: description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore + length for etcd as well as for embedded-etcd of backup-restore sidecar. type: string type: object @@ -1876,13 +1877,13 @@ spec: type: string runID: description: RunID correlates an operation with a reconciliation - run. Every time an etcd resource is reconciled (barring status + run. Every time an Etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering reconcile logs as all structured logs - in a reconcile run should have the `runID` referenced. + in a reconciliation run should have the `runID` referenced. type: string state: description: State is the state of the last operation. @@ -1900,7 +1901,7 @@ spec: members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about a etcd cluster + description: EtcdMemberStatus holds information about etcd cluster membership. properties: id: @@ -1950,7 +1951,7 @@ spec: format: int32 type: integer replicas: - description: Replicas is the replica count of the etcd resource. + description: Replicas is the replica count of the etcd cluster. format: int32 type: integer serviceName: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 6a66b2b81..b09430040 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -339,7 +339,8 @@ spec: type: object type: object etcd: - description: EtcdConfig defines parameters associated etcd deployed + description: EtcdConfig defines the configuration for the etcd cluster + to be deployed. properties: authSecretRef: description: SecretReference represents a Secret Reference. It @@ -1694,7 +1695,7 @@ spec: type: string autoCompactionRetention: description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore + length for etcd as well as for embedded-etcd of backup-restore sidecar. type: string type: object @@ -1876,13 +1877,13 @@ spec: type: string runID: description: RunID correlates an operation with a reconciliation - run. Every time an etcd resource is reconciled (barring status + run. Every time an Etcd resource is reconciled (barring status reconciliation which is periodic), a unique ID is generated which can be used to correlate all actions done as part of a single reconcile run. Capturing this as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering reconcile logs as all structured logs - in a reconcile run should have the `runID` referenced. + in a reconciliation run should have the `runID` referenced. type: string state: description: State is the state of the last operation. @@ -1900,7 +1901,7 @@ spec: members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about a etcd cluster + description: EtcdMemberStatus holds information about etcd cluster membership. properties: id: @@ -1950,7 +1951,7 @@ spec: format: int32 type: integer replicas: - description: Replicas is the replica count of the etcd resource. + description: Replicas is the replica count of the etcd cluster. format: int32 type: integer serviceName: diff --git a/docs/concepts/webhooks.md b/docs/concepts/webhooks.md index 54df0575f..331dcbc09 100644 --- a/docs/concepts/webhooks.md +++ b/docs/concepts/webhooks.md @@ -29,7 +29,7 @@ Unintended changes to any of these *managed resources* can lead to misconfigurat *Sentinel webhook* prevents *UPDATE* and *DELETE* operations on all resources managed by *etcd controller*, unless such an operation is performed by druid itself, and during reconciliation of the `Etcd` resource. Operations are also allowed if performed by one of the authorized entities specified by CLI flag `--sentinel-exempt-service-accounts`. -There may be specific cases where a human operator may need to make changes to the managed resources, possibly to test or fix an etcd cluster. An example of this is [recovery from permanent quorum loss](../operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md), where a human operator will need to suspend reconciliation of the `Etcd` resource, make changes to the underlying managed resources such as `StatefulSet` and `ConfigMap`, and then resume reconciliation for the `Etcd` resource. Such manual interventions will require out-of-band changes to the managed resources. Protection of managed resources for such `Etcd` resources can be turned off by adding an annotation `druid.gardener.cloud/resource-protection: false` on the `Etcd` resource. This will effectively disable *sentinel webhook* protection for all managed resources for the specific `Etcd`. +There may be specific cases where a human operator may need to make changes to the managed resources, possibly to test or fix an etcd cluster. An example of this is [recovery from permanent quorum loss](../operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md), where a human operator will need to suspend reconciliation of the `Etcd` resource, make changes to the underlying managed resources such as `StatefulSet` and `ConfigMap`, and then resume reconciliation for the `Etcd` resource. Such manual interventions will require out-of-band changes to the managed resources. Protection of managed resources for such `Etcd` resources can be turned off by adding an annotation `druid.gardener.cloud/disable-resource-protection` on the `Etcd` resource. This will effectively disable *sentinel webhook* protection for all managed resources for the specific `Etcd`. **Note:** *UPDATE* operations for `Lease`s are always allowed, since these are regularly updated by the etcd-backup-restore sidecar. diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md index b12b0f112..347bbc755 100644 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -21,9 +21,9 @@ If etcd-druid and etcd-backup-restore is being used with gardener, then Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. 1. Add the following annotations to the `Etcd` resource `etcd-main`: - 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile='true'` + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile=` - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection='false'` + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection=` 2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index fed4d2a1a..18ae3fd4e 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -88,7 +88,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } - // allow changes to resources if etcd has annotation druid.gardener.cloud/resource-protection: false + // allow changes to resources if etcd has annotation druid.gardener.cloud/disable-resource-protection is set if !etcd.AreManagedResourcesProtected() { return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } From 42b2765b7b0f18a158d74bc2ad3848151496bb41 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 7 May 2024 14:48:17 +0530 Subject: [PATCH 158/235] Address review comments by @ashwani2k --- docs/concepts/controllers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts/controllers.md b/docs/concepts/controllers.md index c6ad4fec7..2dff0b070 100644 --- a/docs/concepts/controllers.md +++ b/docs/concepts/controllers.md @@ -6,7 +6,7 @@ etcd-druid uses Kubebuilder to define the `Etcd` CR and its corresponding contro All controllers that are a part of etcd-druid reside in package `internal/controller`, as sub-packages. -etcd-druid currently consists of 5 controllers, each having its own responsibility: +etcd-druid currently consists of the following controllers, each having its own responsibility: - *etcd* : responsible for the reconciliation of the `Etcd` CR spec, which allows users to run etcd clusters within the specified Kubernetes cluster, and also responsible for periodically updating the `Etcd` CR status with the up-to-date state of the managed etcd cluster. - *compaction* : responsible for [snapshot compaction](/docs/proposals/02-snapshot-compaction.md). @@ -81,7 +81,7 @@ Status fields related to the etcd cluster itself, such as `Members`, `PeerUrlTLS ## Compaction Controller -The *compaction controller* deploys the snapshot compaction job whenever required. +The *compaction controller* deploys the snapshot compaction job whenever required. To understand the rationale behind this controller, please read [snapshot-compaction.md](../proposals/02-snapshot-compaction.md). The controller watches the number of events accumulated as part of delta snapshots in the etcd cluster's backups, and triggers a snapshot compaction when the number of delta events crosses the set threshold, which is configurable through the `--etcd-events-threshold` CLI flag (1M events by default). The controller watches for changes in *snapshot* `Leases` associated with `Etcd` resources. From f790610b049c6c93353f230d03e898ed0d4d02c4 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 7 May 2024 15:06:12 +0530 Subject: [PATCH 159/235] Address review comments by @renormalize: use constant for volume mode 0640 --- internal/common/constants.go | 3 +++ internal/controller/compaction/config.go | 4 ++-- .../etcdcopybackupstask/reconciler.go | 2 +- .../etcdcopybackupstask/reconciler_test.go | 4 ++-- internal/operator/statefulset/builder.go | 20 +++++++++---------- internal/operator/statefulset/stsmatcher.go | 20 +++++++++---------- test/e2e/utils.go | 13 ++++++------ .../etcdcopybackupstask/reconciler_test.go | 2 +- 8 files changed, 36 insertions(+), 32 deletions(-) diff --git a/internal/common/constants.go b/internal/common/constants.go index 9d51e2872..bcfdd2acc 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -131,6 +131,9 @@ const ( ProviderBackupSecretVolumeName = "etcd-backup-secret" ) +// OwnerReadWriteGroupReadPermissions is the file permissions used for volumes +const OwnerReadWriteGroupReadPermissions int32 = 0640 + // constants for volume mount paths const ( // EtcdCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for client communication are mounted. diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 087510efa..88a13a062 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -24,7 +24,7 @@ const ( workersFlagName = "compaction-workers" eventsThresholdFlagName = "etcd-events-threshold" activeDeadlineDurationFlagName = "active-deadline-duration" - metricsScrapeWaitDurationFlagname = "metrics-scrape-wait-duration" + metricsScrapeWaitDurationFlagName = "metrics-scrape-wait-duration" defaultEnableBackupCompaction = false defaultCompactionWorkers = 3 @@ -59,7 +59,7 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { "Total number of etcd events that can be allowed before a backup compaction job is triggered.") fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, "Duration after which a running backup compaction job will be terminated.") - fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagname, defaultMetricsScrapeWaitDuration, + fs.DurationVar(&cfg.MetricsScrapeWaitDuration, metricsScrapeWaitDurationFlagName, defaultMetricsScrapeWaitDuration, "Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped.") } diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 518692fb1..9a0739f79 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -464,7 +464,7 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }) diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 2dd8861b9..1f982f02a 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -629,7 +629,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Expect(volumeSource.Secret).NotTo(BeNil()) Expect(*volumeSource.Secret).To(Equal(corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), })) }) @@ -953,7 +953,7 @@ func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Ele "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(store.SecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), }), diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 2dd53ba3a..d2e579980 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -702,7 +702,7 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu Path: etcdConfigFileName, }, }, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -737,7 +737,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -746,7 +746,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -755,7 +755,7 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -770,7 +770,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -779,7 +779,7 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -794,7 +794,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -803,7 +803,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -812,7 +812,7 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -851,7 +851,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, nil diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index bdeef5d29..69f28595e 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -421,7 +421,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { "Key": Equal(etcdConfigFileName), "Path": Equal(etcdConfigFileName), })), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), }) @@ -447,7 +447,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -456,7 +456,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -465,7 +465,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -476,7 +476,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -486,7 +486,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -497,7 +497,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -507,7 +507,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -517,7 +517,7 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), })) @@ -550,7 +550,7 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": PointTo(Equal(int32(0640))), + "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), })), }), }) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 7bdcf3b38..cd5df3044 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -15,6 +15,7 @@ import ( "time" "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/etcd-backup-restore/pkg/snapstore" @@ -841,7 +842,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -850,7 +851,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -859,7 +860,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -973,7 +974,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -982,7 +983,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, @@ -991,7 +992,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(0640), + DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), }, }, }, diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index 198c14bdb..ba20c59b6 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -357,7 +357,7 @@ func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Ele "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(0640)), + "DefaultMode": Equal(pointer.Int32(common.OwnerReadWriteGroupReadPermissions)), })), }), }), From cd1565084c150999c44c0d592d456fb475a140f2 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 9 May 2024 09:43:50 +0530 Subject: [PATCH 160/235] addressed review comment to remove explicit types from generic MergeMaps func --- internal/controller/compaction/reconciler.go | 3 ++- internal/operator/clientservice/clientservice.go | 2 +- internal/operator/clientservice/clientservice_test.go | 2 +- internal/operator/configmap/configmap.go | 2 +- internal/operator/configmap/configmap_test.go | 2 +- internal/operator/memberlease/memberlease.go | 4 ++-- internal/operator/memberlease/memberlease_test.go | 2 +- internal/operator/peerservice/peerservice.go | 2 +- .../operator/poddistruptionbudget/poddisruptionbudget.go | 2 +- internal/operator/role/role.go | 2 +- internal/operator/rolebinding/rolebinding.go | 2 +- internal/operator/serviceaccount/serviceaccount.go | 2 +- internal/operator/snapshotlease/snapshotlease.go | 4 ++-- internal/operator/snapshotlease/snapshotlease_test.go | 6 +++--- internal/operator/statefulset/builder.go | 6 +++--- internal/operator/statefulset/stsmatcher.go | 4 ++-- internal/utils/miscellaneous.go | 2 +- 17 files changed, 25 insertions(+), 24 deletions(-) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index d35d242e5..bea1c8734 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -350,8 +350,9 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { "networking.gardener.cloud/to-private-networks": "allowed", "networking.gardener.cloud/to-public-networks": "allowed", } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), jobLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), jobLabels) } + func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featuregate.Feature]bool) ([]v1.VolumeMount, error) { vms := []v1.VolumeMount{ { diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index f70b6541b..6db6b9f3f 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -133,7 +133,7 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { if etcd.Spec.Etcd.ClientService != nil && etcd.Spec.Etcd.ClientService.Labels != nil { specClientSvcLabels = etcd.Spec.Etcd.ClientService.Labels } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), clientSvcLabels, specClientSvcLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), clientSvcLabels, specClientSvcLabels) } func getAnnotations(etcd *druidv1alpha1.Etcd) map[string]string { diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index c3c3efd47..6975d7ea4 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -264,7 +264,7 @@ func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Ser var expectedAnnotations map[string]string if etcd.Spec.Etcd.ClientService != nil { expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations - expectedLabels = utils.MergeMaps[string](etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) + expectedLabels = utils.MergeMaps(etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) } g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 52fb3628e..714a36549 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -128,7 +128,7 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.ConfigMapComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), cmLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), cmLabels) } func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index e20bd3de6..864dd212b 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -313,7 +313,7 @@ func getLatestConfigMap(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Con } func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.ConfigMap) { - expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + expectedLabels := utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ druidv1alpha1.LabelComponentKey: common.ConfigMapComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), }) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 4ca26f12a..3e47b6db1 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -135,7 +135,7 @@ func getSelectorLabelsForAllMemberLeases(etcd *druidv1alpha1.Etcd) map[string]st leaseMatchingLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), leaseMatchingLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { @@ -143,7 +143,7 @@ func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, } - return utils.MergeMaps[string, string](leaseLabels, etcd.GetDefaultLabels()) + return utils.MergeMaps(leaseLabels, etcd.GetDefaultLabels()) } func emptyMemberLease(objectKey client.ObjectKey) *coordinationv1.Lease { diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index e220067c5..dd306fa93 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -273,7 +273,7 @@ func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) return doGetLatestLeases(g, cl, etcd, - utils.MergeMaps[string, string](map[string]string{ + utils.MergeMaps(map[string]string{ druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, }, etcd.GetDefaultLabels())) } diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 3a7f253e2..035d9e673 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -118,7 +118,7 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.PeerServiceComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetPeerServiceName(), } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), svcLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), svcLabels) } func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index 993909161..fdaf6d3e3 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -110,7 +110,7 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.PodDisruptionBudgetComponentName, druidv1alpha1.LabelAppNameKey: etcd.Name, } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), pdbLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), pdbLabels) } func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index a9dfb2284..17d857c65 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -138,5 +138,5 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.RoleComponentName, druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleName(), ":", "-"), // role name contains `:` which is not an allowed character as a label value. } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) } diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index fd0db1447..7431e95f8 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -134,5 +134,5 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.RoleBindingComponentName, druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleBindingName(), ":", "-"), // role-binding name contains `:` which is not an allowed character as a label value. } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) } diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 43887c36c..6c9a921f1 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -110,7 +110,7 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.ServiceAccountComponentName, druidv1alpha1.LabelAppNameKey: etcd.GetServiceAccountName(), } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), roleLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) } func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index a4fc6f8f9..17d02f610 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -166,7 +166,7 @@ func getSelectorLabelsForAllSnapshotLeases(etcd *druidv1alpha1.Etcd) map[string] leaseMatchingLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, } - return utils.MergeMaps[string, string](etcd.GetDefaultLabels(), leaseMatchingLabels) + return utils.MergeMaps(etcd.GetDefaultLabels(), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { @@ -174,7 +174,7 @@ func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, } - return utils.MergeMaps[string, string](leaseLabels, etcd.GetDefaultLabels()) + return utils.MergeMaps(leaseLabels, etcd.GetDefaultLabels()) } func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 4ad911abb..6d827e5c5 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -266,7 +266,7 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas ObjectMeta: metav1.ObjectMeta{ Name: leaseName, Namespace: etcd.Namespace, - Labels: utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + Labels: utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, }), @@ -276,7 +276,7 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas } func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { - expectedLabels := utils.MergeMaps[string, string](etcd.GetDefaultLabels(), map[string]string{ + expectedLabels := utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, druidv1alpha1.LabelAppNameKey: leaseName, }) @@ -293,7 +293,7 @@ func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMa func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coordinationv1.Lease, error) { return doGetLatestLeases(cl, etcd, - utils.MergeMaps[string, string](map[string]string{ + utils.MergeMaps(map[string]string{ druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, }, etcd.GetDefaultLabels())) } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index d2e579980..b2cec0b0e 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -124,7 +124,7 @@ func (b *stsBuilder) getStatefulSetLabels() map[string]string { druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, druidv1alpha1.LabelAppNameKey: b.etcd.Name, } - return utils.MergeMaps[string, string](b.etcd.GetDefaultLabels(), stsLabels) + return utils.MergeMaps(b.etcd.GetDefaultLabels(), stsLabels) } func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error { @@ -149,7 +149,7 @@ func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error ServiceName: b.etcd.GetPeerServiceName(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: utils.MergeMaps[string](b.etcd.Spec.Labels, b.getStatefulSetLabels()), + Labels: utils.MergeMaps(b.etcd.Spec.Labels, b.getStatefulSetLabels()), Annotations: b.getPodTemplateAnnotations(ctx), }, Spec: corev1.PodSpec{ @@ -183,7 +183,7 @@ func (b *stsBuilder) getHostAliases() []corev1.HostAlias { func (b *stsBuilder) getPodTemplateAnnotations(ctx component.OperatorContext) map[string]string { if configMapCheckSum, ok := ctx.Data[common.ConfigMapCheckSumKey]; ok { - return utils.MergeMaps[string](b.etcd.Spec.Annotations, map[string]string{ + return utils.MergeMaps(b.etcd.Spec.Annotations, map[string]string{ common.ConfigMapCheckSumKey: configMapCheckSum, }) } diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 69f28595e..3137b3296 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -132,8 +132,8 @@ func (s StatefulSetMatcher) matchPodTemplateSpec() gomegatypes.GomegaMatcher { func (s StatefulSetMatcher) matchPodObjectMeta() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ - "Labels": testutils.MatchResourceLabels(utils.MergeMaps[string, string](getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), - "Annotations": testutils.MatchResourceAnnotations(utils.MergeMaps[string, string](s.etcd.Spec.Annotations, map[string]string{ + "Labels": testutils.MatchResourceLabels(utils.MergeMaps(getStatefulSetLabels(s.etcd.Name), s.etcd.Spec.Labels)), + "Annotations": testutils.MatchResourceAnnotations(utils.MergeMaps(s.etcd.Spec.Annotations, map[string]string{ "checksum/etcd-configmap": testutils.TestConfigMapCheckSum, })), }) diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index f25db2e16..d6856d15d 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -17,7 +17,7 @@ import ( func MergeStringMaps(oldMap map[string]string, newMaps ...map[string]string) map[string]string { allMaps := []map[string]string{oldMap} allMaps = append(allMaps, newMaps...) - return MergeMaps[string, string](allMaps...) + return MergeMaps(allMaps...) } // MergeMaps merges the contents of maps. All maps will be processed in the order From 109fe5cd8b832ca8955215b80ee264ea3b7a0301 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Thu, 9 May 2024 10:06:41 +0530 Subject: [PATCH 161/235] addressed review comments to streamline constants --- internal/common/constants.go | 11 +++++ .../operator/clientservice/clientservice.go | 13 ++---- .../clientservice/clientservice_test.go | 7 +-- internal/operator/configmap/configmap_test.go | 14 +++--- internal/operator/configmap/etcdconfig.go | 12 ++--- internal/operator/peerservice/peerservice.go | 4 +- .../operator/peerservice/peerservice_test.go | 3 +- internal/operator/statefulset/builder.go | 22 ++++----- test/utils/etcd.go | 46 +++++++++---------- 9 files changed, 65 insertions(+), 67 deletions(-) diff --git a/internal/common/constants.go b/internal/common/constants.go index bcfdd2acc..8d153e05a 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -161,3 +161,14 @@ const ( // EtcdDataVolumeMountPath is the path on a container where the etcd data directory is hosted. EtcdDataVolumeMountPath = "/var/etcd/data" ) + +const ( + // DefaultBackupPort is the default port for the HTTP server in the etcd-backup-restore container. + DefaultBackupPort int32 = 8080 + // DefaultServerPort is the default port for the etcd server used for peer communication. + DefaultServerPort int32 = 2380 + // DefaultClientPort is the default port for the etcd client. + DefaultClientPort int32 = 2379 + // DefaultWrapperPort is the default port for the etcd-wrapper HTTP server. + DefaultWrapperPort int = 9095 +) diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 6db6b9f3f..59f216509 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -20,13 +20,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// default values -const ( - defaultBackupPort = 8080 - defaultClientPort = 2379 - defaultServerPort = 2380 -) - const ( // ErrGetClientService indicates an error in getting the client service resource. ErrGetClientService druidv1alpha1.ErrorCode = "ERR_GET_CLIENT_SERVICE" @@ -153,9 +146,9 @@ func emptyClientService(objectKey client.ObjectKey) *corev1.Service { } func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultBackupPort) + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) return []corev1.ServicePort{ { diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 6975d7ea4..b340f0328 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -9,6 +9,7 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" @@ -256,9 +257,9 @@ func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { } func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort) - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, defaultBackupPort) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort) + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultBackupPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) expectedLabels := etcd.GetDefaultLabels() var expectedAnnotations map[string]string diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 864dd212b..185daee7d 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -352,8 +352,8 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), - "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort))), + "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -364,7 +364,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort))), + "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("client-transport-security")) } @@ -380,13 +380,13 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), "auto-tls": Equal(false), }), - "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))))), + "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))))), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))))), + "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index ada0b8f0d..d635a079a 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -25,8 +25,6 @@ const ( // TODO: Ideally this should be made configurable via Etcd resource as this has a direct impact on the memory requirements for etcd container. // which in turn is influenced by the size of objects that are getting stored in etcd. defaultSnapshotCount = 75000 - defaultClientPort = 2379 - defaultServerPort = 2380 ) var ( @@ -84,10 +82,10 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { InitialCluster: prepareInitialCluster(etcd, peerScheme), AutoCompactionMode: utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), AutoCompactionRetention: utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), - ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), - ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), - AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort)), - AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, defaultClientPort)), + ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort)), + ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort)), + AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort)), + AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort)), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -123,7 +121,7 @@ func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, caPath, serv func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { domainName := fmt.Sprintf("%s.%s.%s", etcd.GetPeerServiceName(), etcd.Namespace, "svc") - serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))) + serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))) builder := strings.Builder{} for i := 0; i < int(etcd.Spec.Replicas); i++ { podName := etcd.GetOrdinalPodName(i) diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 035d9e673..6c84d6243 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -20,8 +20,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const defaultServerPort = 2380 - const ( // ErrGetPeerService indicates an error in getting the peer service resource. ErrGetPeerService druidv1alpha1.ErrorCode = "ERR_GET_PEER_SERVICE" @@ -135,7 +133,7 @@ func emptyPeerService(objectKey client.ObjectKey) *corev1.Service { } func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) return []corev1.ServicePort{ { Name: "peer", diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 86142c294..56a529ac3 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -10,6 +10,7 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" @@ -238,7 +239,7 @@ func newPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { } func matchPeerService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, defaultServerPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(etcd.GetPeerServiceName()), diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index b2cec0b0e..3d0ce4a3a 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -27,9 +27,6 @@ import ( // defaults // ----------------------------------------------------------------------------------------- const ( - defaultBackupPort int32 = 8080 - defaultServerPort int32 = 2380 - defaultClientPort int32 = 2379 defaultWrapperPort int = 9095 defaultMaxBackupsLimitBasedGC int32 = 7 defaultQuota int64 = 8 * 1024 * 1024 * 1024 // 8Gi @@ -42,6 +39,7 @@ const ( defaultAutoCompactionMode = "periodic" defaultEtcdConnectionTimeout = "5m" defaultPodManagementPolicy = appsv1.ParallelPodManagement + nonRootUser = int64(65532) ) var ( @@ -98,9 +96,9 @@ func newStsBuilder(client client.Client, etcdBackupRestoreImage: etcdBackupRestoreImage, initContainerImage: initContainerImage, sts: sts, - clientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, defaultClientPort), - serverPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort), - backupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, defaultBackupPort), + clientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, common.DefaultClientPort), + serverPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort), + backupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, common.DefaultBackupPort), }, nil } @@ -221,7 +219,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 %s", common.EtcdDataVolumeMountPath)}, + Args: []string{fmt.Sprintf("chown -R %d:%d %s", nonRootUser, nonRootUser, common.EtcdDataVolumeMountPath)}, VolumeMounts: []corev1.VolumeMount{b.getEtcdDataVolumeMount()}, SecurityContext: &corev1.SecurityContext{ RunAsGroup: pointer.Int64(0), @@ -238,7 +236,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R 65532:65532 /home/nonroot/%s", *b.etcd.Spec.Backup.Store.Container)}, + Args: []string{fmt.Sprintf("chown -R %d:%d /home/nonroot/%s", nonRootUser, nonRootUser, *b.etcd.Spec.Backup.Store.Container)}, VolumeMounts: []corev1.VolumeMount{*etcdBackupVolumeMount}, SecurityContext: &corev1.SecurityContext{ RunAsGroup: pointer.Int64(0), @@ -565,7 +563,7 @@ func (b *stsBuilder) getEtcdContainerReadinessHandler() corev1.ProbeHandler { } scheme := utils.IfConditionOr[corev1.URIScheme](b.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) path := utils.IfConditionOr[string](multiNodeCluster, "/readyz", "/healthz") - port := utils.IfConditionOr[int](multiNodeCluster, defaultWrapperPort, int(defaultBackupPort)) + port := utils.IfConditionOr[int](multiNodeCluster, defaultWrapperPort, int(common.DefaultBackupPort)) return corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -638,10 +636,10 @@ func (b *stsBuilder) getPodSecurityContext() *corev1.PodSecurityContext { return nil } return &corev1.PodSecurityContext{ - RunAsGroup: pointer.Int64(65532), + RunAsGroup: pointer.Int64(nonRootUser), RunAsNonRoot: pointer.Bool(true), - RunAsUser: pointer.Int64(65532), - FSGroup: pointer.Int64(65532), + RunAsUser: pointer.Int64(nonRootUser), + FSGroup: pointer.Int64(nonRootUser), } } diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 197325d7c..9d449a7ce 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -10,6 +10,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/google/uuid" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -28,27 +29,24 @@ var ( garbageCollectionPeriod = metav1.Duration{ Duration: 43200 * time.Second, } - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - imageEtcd = "eu.gcr.io/gardener-project/gardener/etcd-wrapper:v0.1.0" - imageBR = "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.25.0" - snapshotSchedule = "0 */24 * * *" - defragSchedule = "0 */24 * * *" - container = "default.bkp" - storageCapacity = resource.MustParse("5Gi") - storageClass = "default" - priorityClassName = "class_priority" - deltaSnapShotMemLimit = resource.MustParse("100Mi") - autoCompactionMode = druidv1alpha1.Periodic - autoCompactionRetention = "2m" - quota = resource.MustParse("8Gi") - localProvider = druidv1alpha1.StorageProvider("Local") - prefix = "/tmp" - volumeClaimTemplateName = "etcd-main" - garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) - metricsBasic = druidv1alpha1.Basic - etcdSnapshotTimeout = metav1.Duration{ + imageEtcd = "eu.gcr.io/gardener-project/gardener/etcd-wrapper:v0.1.0" + imageBR = "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.25.0" + snapshotSchedule = "0 */24 * * *" + defragSchedule = "0 */24 * * *" + container = "default.bkp" + storageCapacity = resource.MustParse("5Gi") + storageClass = "default" + priorityClassName = "class_priority" + deltaSnapShotMemLimit = resource.MustParse("100Mi") + autoCompactionMode = druidv1alpha1.Periodic + autoCompactionRetention = "2m" + quota = resource.MustParse("8Gi") + localProvider = druidv1alpha1.StorageProvider("Local") + prefix = "/tmp" + volumeClaimTemplateName = "etcd-main" + garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) + metricsBasic = druidv1alpha1.Basic + etcdSnapshotTimeout = metav1.Duration{ Duration: 10 * time.Minute, } etcdDefragTimeout = metav1.Duration{ @@ -399,8 +397,8 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { "memory": ParseQuantity("1000Mi"), }, }, - ClientPort: &clientPort, - ServerPort: &serverPort, + ClientPort: pointer.Int32(common.DefaultClientPort), + ServerPort: pointer.Int32(common.DefaultServerPort), }, Common: druidv1alpha1.SharedConfig{ AutoCompactionMode: &autoCompactionMode, @@ -413,7 +411,7 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { func getBackupSpec() druidv1alpha1.BackupSpec { return druidv1alpha1.BackupSpec{ Image: &imageBR, - Port: &backupPort, + Port: pointer.Int32(common.DefaultBackupPort), FullSnapshotSchedule: &snapshotSchedule, GarbageCollectionPolicy: &garbageCollectionPolicy, GarbageCollectionPeriod: &garbageCollectionPeriod, From 7c1c8fdc8e3a60d9a4cd308f26edc9d67cb3b333 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 9 May 2024 09:41:06 +0530 Subject: [PATCH 162/235] Address review comments by @renormalize: stylistic changes to bring err population and check into same line --- internal/controller/compaction/reconciler.go | 3 +-- internal/operator/memberlease/memberlease.go | 6 +++--- internal/operator/peerservice/peerservice.go | 3 +-- internal/operator/rolebinding/rolebinding.go | 3 +-- internal/operator/snapshotlease/snapshotlease_test.go | 6 +++--- internal/operator/statefulset/statefulset.go | 3 +-- internal/utils/concurrent.go | 3 +-- test/e2e/etcd_backup_test.go | 3 +-- test/e2e/etcd_multi_node_test.go | 9 +++------ test/e2e/utils.go | 8 +++----- test/it/setup/setup.go | 3 +-- test/utils/secret.go | 9 ++++----- 12 files changed, 23 insertions(+), 36 deletions(-) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index bea1c8734..77d85ed9c 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -148,8 +148,7 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd if job.Status.StartTime != nil { metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(time.Since(job.Status.StartTime.Time).Seconds()) } - err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) - if err != nil { + if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { return ctrl.Result{ RequeueAfter: 10 * time.Second, }, fmt.Errorf("error while deleting failed compaction job: %v", err) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 3e47b6db1..a6ee2b660 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -44,11 +44,11 @@ func New(client client.Client) component.Operator { func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) leaseList := &coordinationv1.LeaseList{} - err := r.client.List(ctx, + if err := r.client.List(ctx, leaseList, client.InNamespace(etcd.Namespace), - client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd))) - if err != nil { + client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd)), + ); err != nil { return resourceNames, druiderr.WrapError(err, ErrListMemberLease, "GetExistingResourceNames", diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 6c84d6243..682db32aa 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -83,8 +83,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of peer service") - err := r.client.Delete(ctx, emptyPeerService(objectKey)) - if err != nil { + if err := r.client.Delete(ctx, emptyPeerService(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No Peer Service found, Deletion is a No-Op", "objectKey", objectKey) return nil diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index 7431e95f8..c9c69ce7a 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -83,8 +83,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) - err := r.client.Delete(ctx, emptyRoleBinding(objectKey)) - if err != nil { + if err := r.client.Delete(ctx, emptyRoleBinding(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No RoleBinding found, Deletion is a No-Op", "objectKey", objectKey) return nil diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index 6d827e5c5..d9db6b2db 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -300,11 +300,11 @@ func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coor func doGetLatestLeases(cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) ([]coordinationv1.Lease, error) { leases := &coordinationv1.LeaseList{} - err := cl.List(context.Background(), + if err := cl.List(context.Background(), leases, client.InNamespace(etcd.Namespace), - client.MatchingLabels(matchingLabels)) - if err != nil { + client.MatchingLabels(matchingLabels), + ); err != nil { return nil, err } return leases.Items, nil diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 4c72f94ed..bbf21db47 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -100,8 +100,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering delete of StatefulSet", "objectKey", objectKey) - err := r.client.Delete(ctx, emptyStatefulSet(etcd)) - if err != nil { + if err := r.client.Delete(ctx, emptyStatefulSet(etcd)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No StatefulSet found, Deletion is a No-Op", "objectKey", objectKey.Name) return nil diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go index 079423a40..aac51e3aa 100644 --- a/internal/utils/concurrent.go +++ b/internal/utils/concurrent.go @@ -61,8 +61,7 @@ func (g *runGroup) trigger(ctx component.OperatorContext, task OperatorTask) { g.errCh <- panicErr } }() - err := task.Fn(ctx) - if err != nil { + if err := task.Fn(ctx); err != nil { g.errCh <- err } }(task) diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index fab74d4a0..a8e1825c3 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -224,8 +224,7 @@ func checkEtcdReady(ctx context.Context, cl client.Client, logger logr.Logger, e ctx, cancelFunc := context.WithTimeout(ctx, timeout) defer cancelFunc() - err := cl.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: namespace}, etcd) - if err != nil { + if err := cl.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: namespace}, etcd); err != nil { return err } diff --git a/test/e2e/etcd_multi_node_test.go b/test/e2e/etcd_multi_node_test.go index 6d6386583..911f73a09 100644 --- a/test/e2e/etcd_multi_node_test.go +++ b/test/e2e/etcd_multi_node_test.go @@ -258,8 +258,7 @@ func checkForUnreadyEtcdMembers(ctx context.Context, cl client.Client, logger lo ctx, cancelFunc := context.WithTimeout(ctx, timeout) defer cancelFunc() - err := cl.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: namespace}, etcd) - if err != nil { + if err := cl.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: namespace}, etcd); err != nil { return err } @@ -335,8 +334,7 @@ func hibernateAndCheckEtcd(ctx context.Context, cl client.Client, logger logr.Lo logger.Info("Checking etcd") EventuallyWithOffset(1, func() error { etcd := getEmptyEtcd(etcd.Name, namespace) - err := cl.Get(ctx, client.ObjectKeyFromObject(etcd), etcd) - if err != nil { + if err := cl.Get(ctx, client.ObjectKeyFromObject(etcd), etcd); err != nil { return err } @@ -503,8 +501,7 @@ func getPodLogs(ctx context.Context, PodKey *types.NamespacedName, opts *corev1. defer podLogs.Close() buf := new(bytes.Buffer) - _, err = io.Copy(buf, podLogs) - if err != nil { + if _, err = io.Copy(buf, podLogs); err != nil { return "", err } return buf.String(), nil diff --git a/test/e2e/utils.go b/test/e2e/utils.go index cd5df3044..d918bcd2e 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -585,11 +585,10 @@ func executeRemoteCommand(ctx context.Context, kubeconfigPath, namespace, podNam buf := &bytes.Buffer{} errBuf := &bytes.Buffer{} - err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + if err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ Stdout: buf, Stderr: errBuf, - }) - if err != nil { + }); err != nil { return "", "", err } @@ -617,8 +616,7 @@ func purgeSnapstore(store brtypes.SnapStore) error { } for _, snap := range snapList { - err = store.Delete(*snap) - if err != nil { + if err = store.Delete(*snap); err != nil { return err } } diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index cbebb9a40..b44e2c02e 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -70,8 +70,7 @@ func NewIntegrationTestEnv(loggerName string, crdDirectoryPaths []string) (Integ } return itEnv, func() { itEnv.cancelFn() - err := itEnv.testEnv.Stop() - if err != nil { + if err := itEnv.testEnv.Stop(); err != nil { logger.Error(err, "failed to stop test environment") } logger.Info("stopped test environment") diff --git a/test/utils/secret.go b/test/utils/secret.go index 87ecb0c69..6c2de3cab 100644 --- a/test/utils/secret.go +++ b/test/utils/secret.go @@ -27,11 +27,10 @@ func CreateSecrets(ctx context.Context, c client.Client, namespace string, secre "test": []byte("test"), }, } - err := c.Create(ctx, &secret) - if apierrors.IsAlreadyExists(err) { - continue - } - if err != nil { + if err := c.Create(ctx, &secret); err != nil { + if apierrors.IsAlreadyExists(err) { + continue + } createErrs = errors.Join(createErrs, err) } } From 0105b2db32e3e95a04808a268c7b48d41e4e2378 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 9 May 2024 11:05:17 +0530 Subject: [PATCH 163/235] Rename all common constants to format --- internal/common/constants.go | 226 +++++++++--------- internal/controller/compaction/reconciler.go | 14 +- .../etcdcopybackupstask/reconciler.go | 14 +- .../etcdcopybackupstask/reconciler_test.go | 32 +-- internal/health/etcdmember/check_ready.go | 2 +- .../health/etcdmember/check_ready_test.go | 2 +- .../operator/clientservice/clientservice.go | 8 +- .../clientservice/clientservice_test.go | 6 +- internal/operator/configmap/configmap.go | 4 +- internal/operator/configmap/configmap_test.go | 18 +- internal/operator/configmap/etcdconfig.go | 16 +- internal/operator/memberlease/memberlease.go | 4 +- .../operator/memberlease/memberlease_test.go | 4 +- internal/operator/peerservice/peerservice.go | 4 +- .../operator/peerservice/peerservice_test.go | 2 +- .../poddisruptionbudget.go | 2 +- internal/operator/role/role.go | 2 +- internal/operator/rolebinding/rolebinding.go | 2 +- .../operator/serviceaccount/serviceaccount.go | 2 +- .../operator/snapshotlease/snapshotlease.go | 4 +- .../snapshotlease/snapshotlease_test.go | 6 +- internal/operator/statefulset/builder.go | 151 ++++++------ internal/operator/statefulset/statefulset.go | 4 +- .../operator/statefulset/statefulset_test.go | 2 +- internal/operator/statefulset/stsmatcher.go | 84 +++---- internal/utils/envvar.go | 12 +- internal/utils/image.go | 12 +- internal/utils/image_test.go | 18 +- internal/utils/lease.go | 2 +- internal/utils/lease_test.go | 4 +- test/e2e/utils.go | 12 +- .../controllers/compaction/reconciler_test.go | 34 +-- .../etcdcopybackupstask/reconciler_test.go | 22 +- test/it/controller/etcd/reconciler_test.go | 2 +- test/utils/etcd.go | 6 +- test/utils/etcdcopybackupstask.go | 2 +- test/utils/imagevector.go | 10 +- test/utils/statefulset.go | 4 +- 38 files changed, 377 insertions(+), 378 deletions(-) diff --git a/internal/common/constants.go b/internal/common/constants.go index 8d153e05a..de47b08c2 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -4,25 +4,53 @@ package common -// Constants for image keys -const ( - // Etcd is the key for the etcd image in the image vector. - Etcd = "etcd" - // BackupRestore is the key for the etcd-backup-restore image in the image vector. - BackupRestore = "etcd-backup-restore" - // EtcdWrapper is the key for the etcd image in the image vector. - EtcdWrapper = "etcd-wrapper" - // BackupRestoreDistroless is the key for the etcd-backup-restore image in the image vector. - BackupRestoreDistroless = "etcd-backup-restore-distroless" - // Alpine is the key for the alpine image in the image vector. - Alpine = "alpine" -) - const ( // DefaultImageVectorFilePath is the path to the default image vector file. DefaultImageVectorFilePath = "charts/images.yaml" // FinalizerName is the name of the etcd finalizer. FinalizerName = "druid.gardener.cloud/etcd-druid" + // CheckSumKeyConfigMap is the key that is set by a configmap operator and used by StatefulSet operator to + // place an annotation on the StatefulSet pods. The value contains the check-sum of the latest configmap that + // should be reflected on the pods. + CheckSumKeyConfigMap = "checksum/etcd-configmap" +) + +// Constants for image keys +const ( + // ImageKeyEtcd is the key for the etcd image in the image vector. + ImageKeyEtcd = "etcd" + // ImageKeyEtcdBackupRestore is the key for the etcd-backup-restore image in the image vector. + ImageKeyEtcdBackupRestore = "etcd-backup-restore" + // ImageKeyEtcdWrapper is the key for the etcd image in the image vector. + ImageKeyEtcdWrapper = "etcd-wrapper" + // ImageKeyEtcdBackupRestoreDistroless is the key for the etcd-backup-restore image in the image vector. + ImageKeyEtcdBackupRestoreDistroless = "etcd-backup-restore-distroless" + // ImageKeyAlpine is the key for the alpine image in the image vector. + ImageKeyAlpine = "alpine" +) + +// Constants for container names +const ( + // ContainerNameEtcd is the name of the etcd container. + ContainerNameEtcd = "etcd" + // ContainerNameEtcdBackupRestore is the name of the backup-restore container. + ContainerNameEtcdBackupRestore = "backup-restore" + // InitContainerNameChangePermissions is the name of the change permissions init container. + InitContainerNameChangePermissions = "change-permissions" + // InitContainerNameChangeBackupBucketPermissions is the name of the change backup bucket permissions init container. + InitContainerNameChangeBackupBucketPermissions = "change-backup-bucket-permissions" +) + +// Constants for ports +const ( + // DefaultPortEtcdPeer is the default port for the etcd server used for peer communication. + DefaultPortEtcdPeer int32 = 2380 + // DefaultPortEtcdClient is the default port for the etcd client. + DefaultPortEtcdClient int32 = 2379 + // DefaultPortEtcdWrapper is the default port for the etcd-wrapper HTTP server. + DefaultPortEtcdWrapper int32 = 9095 + // DefaultPortEtcdBackupRestore is the default port for the HTTP server in the etcd-backup-restore container. + DefaultPortEtcdBackupRestore int32 = 8080 ) // Constants for environment variables @@ -59,116 +87,86 @@ const ( // Constants for values to be set against druidv1alpha1.LabelComponentKey const ( - // ClientServiceComponentName is the component name for client service resource. - ClientServiceComponentName = "etcd-client-service" - // ConfigMapComponentName is the component name for config map resource. - ConfigMapComponentName = "etcd-config" - // MemberLeaseComponentName is the component name for member lease resource. - MemberLeaseComponentName = "etcd-member-lease" - // SnapshotLeaseComponentName is the component name for snapshot lease resource. - SnapshotLeaseComponentName = "etcd-snapshot-lease" - // PeerServiceComponentName is the component name for peer service resource. - PeerServiceComponentName = "etcd-peer-service" - // PodDisruptionBudgetComponentName is the component name for pod disruption budget resource. - PodDisruptionBudgetComponentName = "etcd-pdb" - // RoleComponentName is the component name for role resource. - RoleComponentName = "etcd-druid-role" - // RoleBindingComponentName is the component name for role binding resource. - RoleBindingComponentName = "druid-role-binding" - // ServiceAccountComponentName is the component name for service account resource. - ServiceAccountComponentName = "druid-service-account" - // StatefulSetComponentName is the component name for statefulset resource. - StatefulSetComponentName = "etcd-sts" - // CompactionJobComponentName is the component name for compaction job resource. - CompactionJobComponentName = "etcd-compaction-job" - // EtcdCopyBackupTaskComponentName is the component name for copy-backup task resource. - EtcdCopyBackupTaskComponentName = "etcd-copy-backup-task" -) - -const ( - // ConfigMapCheckSumKey is the key that is set by a configmap operator and used by StatefulSet operator to - // place an annotation on the StatefulSet pods. The value contains the check-sum of the latest configmap that - // should be reflected on the pods. - ConfigMapCheckSumKey = "checksum/etcd-configmap" -) - -// Constants for container names -const ( - // EtcdContainerName is the name of the etcd container. - EtcdContainerName = "etcd" - // EtcdBackupRestoreContainerName is the name of the backup-restore container. - EtcdBackupRestoreContainerName = "backup-restore" - // ChangePermissionsInitContainerName is the name of the change permissions init container. - ChangePermissionsInitContainerName = "change-permissions" - // ChangeBackupBucketPermissionsInitContainerName is the name of the change backup bucket permissions init container. - ChangeBackupBucketPermissionsInitContainerName = "change-backup-bucket-permissions" + // ComponentNameClientService is the component name for client service resource. + ComponentNameClientService = "etcd-client-service" + // ComponentNameConfigMap is the component name for config map resource. + ComponentNameConfigMap = "etcd-config" + // ComponentNameMemberLease is the component name for member lease resource. + ComponentNameMemberLease = "etcd-member-lease" + // ComponentNameSnapshotLease is the component name for snapshot lease resource. + ComponentNameSnapshotLease = "etcd-snapshot-lease" + // ComponentNamePeerService is the component name for peer service resource. + ComponentNamePeerService = "etcd-peer-service" + // ComponentNamePodDisruptionBudget is the component name for pod disruption budget resource. + ComponentNamePodDisruptionBudget = "etcd-pdb" + // ComponentNameRole is the component name for role resource. + ComponentNameRole = "etcd-druid-role" + // ComponentNameRoleBinding is the component name for role binding resource. + ComponentNameRoleBinding = "druid-role-binding" + // ComponentNameServiceAccount is the component name for service account resource. + ComponentNameServiceAccount = "druid-service-account" + // ComponentNameStatefulSet is the component name for statefulset resource. + ComponentNameStatefulSet = "etcd-sts" + // ComponentNameCompactionJob is the component name for compaction job resource. + ComponentNameCompactionJob = "etcd-compaction-job" + // ComponentNameEtcdCopyBackupsTask is the component name for copy-backup task resource. + ComponentNameEtcdCopyBackupsTask = "etcd-copy-backup-task" ) // Constants for volume names const ( - // EtcdCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. - EtcdCAVolumeName = "etcd-ca" - // EtcdServerTLSVolumeName is the name of the volume that contains the server certificate-key pair used to set up the etcd server and etcd-wrapper HTTP server. - EtcdServerTLSVolumeName = "etcd-server-tls" - // EtcdClientTLSVolumeName is the name of the volume that contains the client certificate-key pair used by the client to communicate to the etcd server and etcd-wrapper HTTP server. - EtcdClientTLSVolumeName = "etcd-client-tls" - // EtcdPeerCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for peer communication. - EtcdPeerCAVolumeName = "etcd-peer-ca" - // EtcdPeerServerTLSVolumeName is the name of the volume that contains the server certificate-key pair used to set up the peer server. - EtcdPeerServerTLSVolumeName = "etcd-peer-server-tls" - // BackupRestoreCAVolumeName is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for backup-restore communication. - BackupRestoreCAVolumeName = "backup-restore-ca" - // BackupRestoreServerTLSVolumeName is the name of the volume that contains the server certificate-key pair used to set up the backup-restore server. - BackupRestoreServerTLSVolumeName = "backup-restore-server-tls" - // BackupRestoreClientTLSVolumeName is the name of the volume that contains the client certificate-key pair used by the client to communicate to the backup-restore server. - BackupRestoreClientTLSVolumeName = "backup-restore-client-tls" + // VolumeNameEtcdCA is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for client communication. + VolumeNameEtcdCA = "etcd-ca" + // VolumeNameEtcdServerTLS is the name of the volume that contains the server certificate-key pair used to set up the etcd server and etcd-wrapper HTTP server. + VolumeNameEtcdServerTLS = "etcd-server-tls" + // VolumeNameEtcdClientTLS is the name of the volume that contains the client certificate-key pair used by the client to communicate to the etcd server and etcd-wrapper HTTP server. + VolumeNameEtcdClientTLS = "etcd-client-tls" + // VolumeNameEtcdPeerCA is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for peer communication. + VolumeNameEtcdPeerCA = "etcd-peer-ca" + // VolumeNameEtcdPeerServerTLS is the name of the volume that contains the server certificate-key pair used to set up the peer server. + VolumeNameEtcdPeerServerTLS = "etcd-peer-server-tls" + // VolumeNameBackupRestoreCA is the name of the volume that contains the CA certificate bundle and CA certificate key used to sign certificates for backup-restore communication. + VolumeNameBackupRestoreCA = "backup-restore-ca" + // VolumeNameBackupRestoreServerTLS is the name of the volume that contains the server certificate-key pair used to set up the backup-restore server. + VolumeNameBackupRestoreServerTLS = "backup-restore-server-tls" + // VolumeNameBackupRestoreClientTLS is the name of the volume that contains the client certificate-key pair used by the client to communicate to the backup-restore server. + VolumeNameBackupRestoreClientTLS = "backup-restore-client-tls" - // EtcdConfigVolumeName is the name of the volume that contains the etcd configuration file. - EtcdConfigVolumeName = "etcd-config-file" - // LocalBackupVolumeName is the name of the volume that contains the local backup. - LocalBackupVolumeName = "local-backup" - // ProviderBackupSecretVolumeName is the name of the volume that contains the provider backup secret. - ProviderBackupSecretVolumeName = "etcd-backup-secret" + // VolumeNameEtcdConfig is the name of the volume that contains the etcd configuration file. + VolumeNameEtcdConfig = "etcd-config-file" + // VolumeNameLocalBackup is the name of the volume that contains the local backup. + VolumeNameLocalBackup = "local-backup" + // VolumeNameProviderBackupSecret is the name of the volume that contains the provider backup secret. + VolumeNameProviderBackupSecret = "etcd-backup-secret" ) -// OwnerReadWriteGroupReadPermissions is the file permissions used for volumes -const OwnerReadWriteGroupReadPermissions int32 = 0640 +// ModeOwnerReadWriteGroupRead is the file permissions used for volumes +const ModeOwnerReadWriteGroupRead int32 = 0640 // constants for volume mount paths const ( - // EtcdCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for client communication are mounted. - EtcdCAVolumeMountPath = "/var/etcd/ssl/ca" - // EtcdServerTLSVolumeMountPath is the path on a container where the server certificate-key pair used to set up the etcd server and etcd-wrapper HTTP server is mounted. - EtcdServerTLSVolumeMountPath = "/var/etcd/ssl/server" - // EtcdClientTLSVolumeMountPath is the path on a container where the client certificate-key pair used by the client to communicate to the etcd server and etcd-wrapper HTTP server is mounted. - EtcdClientTLSVolumeMountPath = "/var/etcd/ssl/client" - // EtcdPeerCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for peer communication are mounted. - EtcdPeerCAVolumeMountPath = "/var/etcd/ssl/peer/ca" - // EtcdPeerServerTLSVolumeMountPath is the path on a container where the server certificate-key pair used to set up the peer server is mounted. - EtcdPeerServerTLSVolumeMountPath = "/var/etcd/ssl/peer/server" - // BackupRestoreCAVolumeMountPath is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for backup-restore communication are mounted. - BackupRestoreCAVolumeMountPath = "/var/etcdbr/ssl/ca" - // BackupRestoreServerTLSVolumeMountPath is the path on a container where the server certificate-key pair used to set up the backup-restore server is mounted. - BackupRestoreServerTLSVolumeMountPath = "/var/etcdbr/ssl/server" - // BackupRestoreClientTLSVolumeMountPath is the path on a container where the client certificate-key pair used by the client to communicate to the backup-restore server is mounted. - BackupRestoreClientTLSVolumeMountPath = "/var/etcdbr/ssl/client" + // VolumeMountPathEtcdCA is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for client communication are mounted. + VolumeMountPathEtcdCA = "/var/etcd/ssl/ca" + // VolumeMountPathEtcdServerTLS is the path on a container where the server certificate-key pair used to set up the etcd server and etcd-wrapper HTTP server is mounted. + VolumeMountPathEtcdServerTLS = "/var/etcd/ssl/server" + // VolumeMountPathEtcdClientTLS is the path on a container where the client certificate-key pair used by the client to communicate to the etcd server and etcd-wrapper HTTP server is mounted. + VolumeMountPathEtcdClientTLS = "/var/etcd/ssl/client" + // VolumeMountPathEtcdPeerCA is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for peer communication are mounted. + VolumeMountPathEtcdPeerCA = "/var/etcd/ssl/peer/ca" + // VolumeMountPathEtcdPeerServerTLS is the path on a container where the server certificate-key pair used to set up the peer server is mounted. + VolumeMountPathEtcdPeerServerTLS = "/var/etcd/ssl/peer/server" + // VolumeMountPathBackupRestoreCA is the path on a container where the CA certificate bundle and CA certificate key used to sign certificates for backup-restore communication are mounted. + VolumeMountPathBackupRestoreCA = "/var/etcdbr/ssl/ca" + // VolumeMountPathBackupRestoreServerTLS is the path on a container where the server certificate-key pair used to set up the backup-restore server is mounted. + VolumeMountPathBackupRestoreServerTLS = "/var/etcdbr/ssl/server" + // VolumeMountPathBackupRestoreClientTLS is the path on a container where the client certificate-key pair used by the client to communicate to the backup-restore server is mounted. + VolumeMountPathBackupRestoreClientTLS = "/var/etcdbr/ssl/client" - // GCSBackupSecretVolumeMountPath is the path on a container where the GCS backup secret is mounted. - GCSBackupSecretVolumeMountPath = "/var/.gcp/" - // NonGCSProviderBackupSecretVolumeMountPath is the path on a container where the non-GCS provider backup secret is mounted. - NonGCSProviderBackupSecretVolumeMountPath = "/var/etcd-backup" + // VolumeMountPathGCSBackupSecret is the path on a container where the GCS backup secret is mounted. + VolumeMountPathGCSBackupSecret = "/var/.gcp/" + // VolumeMountPathNonGCSProviderBackupSecret is the path on a container where the non-GCS provider backup secret is mounted. + VolumeMountPathNonGCSProviderBackupSecret = "/var/etcd-backup" - // EtcdDataVolumeMountPath is the path on a container where the etcd data directory is hosted. - EtcdDataVolumeMountPath = "/var/etcd/data" -) - -const ( - // DefaultBackupPort is the default port for the HTTP server in the etcd-backup-restore container. - DefaultBackupPort int32 = 8080 - // DefaultServerPort is the default port for the etcd server used for peer communication. - DefaultServerPort int32 = 2380 - // DefaultClientPort is the default port for the etcd client. - DefaultClientPort int32 = 2379 - // DefaultWrapperPort is the default port for the etcd-wrapper HTTP server. - DefaultWrapperPort int = 9095 + // VolumeMountPathEtcdData is the path on a container where the etcd data directory is hosted. + VolumeMountPathEtcdData = "/var/etcd/data" ) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 77d85ed9c..dda0e8ece 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -344,7 +344,7 @@ func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { jobLabels := map[string]string{ druidv1alpha1.LabelAppNameKey: etcd.GetCompactionJobName(), - druidv1alpha1.LabelComponentKey: common.CompactionJobComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameCompactionJob, "networking.gardener.cloud/to-dns": "allowed", "networking.gardener.cloud/to-private-networks": "allowed", "networking.gardener.cloud/to-public-networks": "allowed", @@ -356,7 +356,7 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu vms := []v1.VolumeMount{ { Name: "etcd-workspace-dir", - MountPath: common.EtcdDataVolumeMountPath, + MountPath: common.VolumeMountPathEtcdData, }, } @@ -379,13 +379,13 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu } case utils.GCS: vms = append(vms, v1.VolumeMount{ - Name: common.ProviderBackupSecretVolumeName, - MountPath: common.GCSBackupSecretVolumeMountPath, + Name: common.VolumeNameProviderBackupSecret, + MountPath: common.VolumeMountPathGCSBackupSecret, }) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: vms = append(vms, v1.VolumeMount{ - Name: common.ProviderBackupSecretVolumeName, - MountPath: common.NonGCSProviderBackupSecretVolumeMountPath, + Name: common.VolumeNameProviderBackupSecret, + MountPath: common.VolumeMountPathNonGCSProviderBackupSecret, }) } @@ -430,7 +430,7 @@ func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr. } vs = append(vs, v1.Volume{ - Name: common.ProviderBackupSecretVolumeName, + Name: common.VolumeNameProviderBackupSecret, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: storeValues.SecretRef.Name, diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 9a0739f79..e2eb1b7ad 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -387,7 +387,7 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et func getLabels(task *druidv1alpha1.EtcdCopyBackupsTask, includeNetworkPolicyLabels bool) map[string]string { labels := make(map[string]string) - labels[druidv1alpha1.LabelComponentKey] = common.EtcdCopyBackupTaskComponentName + labels[druidv1alpha1.LabelComponentKey] = common.ComponentNameEtcdCopyBackupsTask labels[druidv1alpha1.LabelPartOfKey] = task.Name labels[druidv1alpha1.LabelManagedByKey] = druidv1alpha1.LabelManagedByValue labels[druidv1alpha1.LabelAppNameKey] = task.GetJobName() @@ -460,11 +460,11 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a return } volumes = append(volumes, corev1.Volume{ - Name: getVolumeNamePrefix(prefix) + common.ProviderBackupSecretVolumeName, + Name: getVolumeNamePrefix(prefix) + common.VolumeNameProviderBackupSecret, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }) @@ -492,12 +492,12 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum } case utils.GCS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, + Name: getVolumeNamePrefix(volumeMountPrefix) + common.VolumeNameProviderBackupSecret, MountPath: getGCSSecretVolumeMountPathWithPrefixAndSuffix(getVolumeNamePrefix(volumeMountPrefix), "/"), }) case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getVolumeNamePrefix(volumeMountPrefix) + common.ProviderBackupSecretVolumeName, + Name: getVolumeNamePrefix(volumeMountPrefix) + common.VolumeNameProviderBackupSecret, MountPath: getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), }) } @@ -506,13 +506,13 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum func getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { // "/var/etcd-backup" - tokens := strings.Split(strings.Trim(common.NonGCSProviderBackupSecretVolumeMountPath, "/"), "/") + tokens := strings.Split(strings.Trim(common.VolumeMountPathNonGCSProviderBackupSecret, "/"), "/") return fmt.Sprintf("/%s/%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) } func getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { // "/var/.gcp" - tokens := strings.Split(strings.TrimSuffix(common.GCSBackupSecretVolumeMountPath, "/"), ".") + tokens := strings.Split(strings.TrimSuffix(common.VolumeMountPathGCSBackupSecret, "/"), ".") return fmt.Sprintf("%s.%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) } diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 1f982f02a..0689d51eb 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -163,12 +163,12 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { logger: logr.Discard(), imageVector: imagevector.ImageVector{ &imagevector.ImageSource{ - Name: common.BackupRestore, + Name: common.ImageKeyEtcdBackupRestore, Repository: "test-repo", Tag: pointer.String("etcd-test-tag"), }, &imagevector.ImageSource{ - Name: common.Alpine, + Name: common.ImageKeyAlpine, Repository: "test-repo", Tag: pointer.String("init-container-test-tag"), }, @@ -471,10 +471,10 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { expectedMountName = volumeMountPrefix + "host-storage" expectedMountPath = *storeSpec.Container case utils.GCS: - expectedMountName = volumeMountPrefix + common.ProviderBackupSecretVolumeName + expectedMountName = volumeMountPrefix + common.VolumeNameProviderBackupSecret expectedMountPath = getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: - expectedMountName = volumeMountPrefix + common.ProviderBackupSecretVolumeName + expectedMountName = volumeMountPrefix + common.VolumeNameProviderBackupSecret expectedMountPath = getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") default: Fail(fmt.Sprintf("Unknown provider: %s", provider)) @@ -629,7 +629,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Expect(volumeSource.Secret).NotTo(BeNil()) Expect(*volumeSource.Secret).To(Equal(corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), })) }) @@ -724,16 +724,16 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I targetProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) Expect(err).NotTo(HaveOccurred()) - images, err := imagevector.FindImages(imageVector, []string{common.BackupRestore}) + images, err := imagevector.FindImages(imageVector, []string{common.ImageKeyEtcdBackupRestore}) Expect(err).NotTo(HaveOccurred()) - backupRestoreImage := images[common.BackupRestore] + backupRestoreImage := images[common.ImageKeyEtcdBackupRestore] matcher := MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), @@ -753,7 +753,7 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Template": MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), @@ -931,15 +931,15 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { switch storeProvider { case "GCS": return Elements{ - volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(getVolumeNamePrefix(volumePrefix) + common.ProviderBackupSecretVolumeName), + volumePrefix + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(getVolumeNamePrefix(volumePrefix) + common.VolumeNameProviderBackupSecret), "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), }), } default: return Elements{ - volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), + volumePrefix + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.VolumeNameProviderBackupSecret), "MountPath": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } @@ -948,12 +948,12 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Elements { return Elements{ - volumePrefix + common.ProviderBackupSecretVolumeName: MatchAllFields(Fields{ - "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), + volumePrefix + common.VolumeNameProviderBackupSecret: MatchAllFields(Fields{ + "Name": Equal(volumePrefix + common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(store.SecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), }), diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index be3c03fcd..3ab0b9541 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -37,7 +37,7 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul leases := &coordinationv1.LeaseList{} if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: etcd.Name, }); err != nil { diff --git a/internal/health/etcdmember/check_ready_test.go b/internal/health/etcdmember/check_ready_test.go index 15865e953..8686e4235 100644 --- a/internal/health/etcdmember/check_ready_test.go +++ b/internal/health/etcdmember/check_ready_test.go @@ -76,7 +76,7 @@ var _ = Describe("ReadyCheck", func() { JustBeforeEach(func() { cl.EXPECT().List(ctx, gomock.AssignableToTypeOf(&coordinationv1.LeaseList{}), client.InNamespace(etcd.Namespace), client.MatchingLabels{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: etcd.Name, }). diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 59f216509..d3b321b89 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -119,7 +119,7 @@ func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { clientSvcLabels := map[string]string{ druidv1alpha1.LabelAppNameKey: etcd.GetClientServiceName(), - druidv1alpha1.LabelComponentKey: common.ClientServiceComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameClientService, } // Add any client service labels as defined in the etcd resource specClientSvcLabels := map[string]string{} @@ -146,9 +146,9 @@ func emptyClientService(objectKey client.ObjectKey) *corev1.Service { } func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultBackupPort) - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) return []corev1.ServicePort{ { diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index b340f0328..49db58bcc 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -257,9 +257,9 @@ func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { } func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort) - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultBackupPort) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) + clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) + backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) expectedLabels := etcd.GetDefaultLabels() var expectedAnnotations map[string]string diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index 714a36549..f0547866a 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -83,7 +83,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) "Sync", fmt.Sprintf("Error when computing CheckSum for configmap for etcd: %v", etcd.GetNamespaceName())) } - ctx.Data[common.ConfigMapCheckSumKey] = checkSum + ctx.Data[common.CheckSumKeyConfigMap] = checkSum ctx.Logger.Info("synced", "component", "configmap", "name", cm.Name, "result", result) return nil } @@ -125,7 +125,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { cmLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.ConfigMapComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameConfigMap, druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), } return utils.MergeMaps(etcd.GetDefaultLabels(), cmLabels) diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index 185daee7d..eb95bc36d 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -314,7 +314,7 @@ func getLatestConfigMap(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Con func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.ConfigMap) { expectedLabels := utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ - druidv1alpha1.LabelComponentKey: common.ConfigMapComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameConfigMap, druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), }) g.Expect(actualConfigMap).To(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ @@ -335,7 +335,7 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C g.Expect(err).ToNot(HaveOccurred()) g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "name": Equal(fmt.Sprintf("etcd-%s", etcd.UID[:6])), - "data-dir": Equal(fmt.Sprintf("%s/new.etcd", common.EtcdDataVolumeMountPath)), + "data-dir": Equal(fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData)), "metrics": Equal(string(druidv1alpha1.Basic)), "snapshot-count": Equal(int64(75000)), "enable-v2": Equal(false), @@ -352,8 +352,8 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort))), - "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort))), + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -364,7 +364,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort))), + "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("client-transport-security")) } @@ -380,13 +380,13 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), "auto-tls": Equal(false), }), - "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))))), + "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))))), + "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index d635a079a..38ec83e93 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -28,7 +28,7 @@ const ( ) var ( - defaultDataDir = fmt.Sprintf("%s/new.etcd", common.EtcdDataVolumeMountPath) + defaultDataDir = fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData) ) type tlsTarget string @@ -67,8 +67,8 @@ type securityConfig struct { } func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { - clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, common.EtcdCAVolumeMountPath, common.EtcdServerTLSVolumeMountPath) - peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, common.EtcdPeerCAVolumeMountPath, common.EtcdPeerServerTLSVolumeMountPath) + clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, common.VolumeMountPathEtcdCA, common.VolumeMountPathEtcdServerTLS) + peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, common.VolumeMountPathEtcdPeerCA, common.VolumeMountPathEtcdPeerServerTLS) cfg := &etcdConfig{ Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), @@ -82,10 +82,10 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { InitialCluster: prepareInitialCluster(etcd, peerScheme), AutoCompactionMode: utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), AutoCompactionRetention: utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), - ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort)), - ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort)), - AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort)), - AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultClientPort)), + ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), + ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), + AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), + AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -121,7 +121,7 @@ func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, caPath, serv func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { domainName := fmt.Sprintf("%s.%s.%s", etcd.GetPeerServiceName(), etcd.Namespace, "svc") - serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort))) + serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))) builder := strings.Builder{} for i := 0; i < int(etcd.Spec.Replicas); i++ { podName := etcd.GetOrdinalPodName(i) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index a6ee2b660..50c4fb5e4 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -133,14 +133,14 @@ func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { func getSelectorLabelsForAllMemberLeases(etcd *druidv1alpha1.Etcd) map[string]string { leaseMatchingLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, } return utils.MergeMaps(etcd.GetDefaultLabels(), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { leaseLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelAppNameKey: leaseName, } return utils.MergeMaps(leaseLabels, etcd.GetDefaultLabels()) diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/operator/memberlease/memberlease_test.go index dd306fa93..077304325 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/operator/memberlease/memberlease_test.go @@ -247,7 +247,7 @@ func TestTriggerDelete(t *testing.T) { } } for _, nonTargetLeaseName := range nonTargetLeaseNames { - existingObjects = append(existingObjects, testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.MemberLeaseComponentName)) + existingObjects = append(existingObjects, testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.ComponentNameMemberLease)) } cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd), existingObjects...) // ***************** Setup operator and test ***************** @@ -274,7 +274,7 @@ func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) cl, etcd, utils.MergeMaps(map[string]string{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, }, etcd.GetDefaultLabels())) } diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 682db32aa..761f7a5e9 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -112,7 +112,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { svcLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.PeerServiceComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNamePeerService, druidv1alpha1.LabelAppNameKey: etcd.GetPeerServiceName(), } return utils.MergeMaps(etcd.GetDefaultLabels(), svcLabels) @@ -132,7 +132,7 @@ func emptyPeerService(objectKey client.ObjectKey) *corev1.Service { } func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) return []corev1.ServicePort{ { Name: "peer", diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index 56a529ac3..e957df624 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -239,7 +239,7 @@ func newPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { } func matchPeerService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultServerPort) + peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(etcd.GetPeerServiceName()), diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index fdaf6d3e3..bea2ed3fc 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -107,7 +107,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { pdbLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.PodDisruptionBudgetComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNamePodDisruptionBudget, druidv1alpha1.LabelAppNameKey: etcd.Name, } return utils.MergeMaps(etcd.GetDefaultLabels(), pdbLabels) diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index 17d857c65..bb3bba6c9 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -135,7 +135,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.RoleComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameRole, druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleName(), ":", "-"), // role name contains `:` which is not an allowed character as a label value. } return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index c9c69ce7a..23abb37c2 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -130,7 +130,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.RoleBindingComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameRoleBinding, druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleBindingName(), ":", "-"), // role-binding name contains `:` which is not an allowed character as a label value. } return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 6c9a921f1..4e173c919 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -107,7 +107,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, sa *corev1.ServiceAccount, autoMoun func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.ServiceAccountComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameServiceAccount, druidv1alpha1.LabelAppNameKey: etcd.GetServiceAccountName(), } return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 17d02f610..40da20154 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -164,14 +164,14 @@ func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { func getSelectorLabelsForAllSnapshotLeases(etcd *druidv1alpha1.Etcd) map[string]string { leaseMatchingLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, } return utils.MergeMaps(etcd.GetDefaultLabels(), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { leaseLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelAppNameKey: leaseName, } return utils.MergeMaps(leaseLabels, etcd.GetDefaultLabels()) diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/operator/snapshotlease/snapshotlease_test.go index d9db6b2db..abedca6f6 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/operator/snapshotlease/snapshotlease_test.go @@ -267,7 +267,7 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas Name: leaseName, Namespace: etcd.Namespace, Labels: utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ - druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelAppNameKey: leaseName, }), OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, @@ -277,7 +277,7 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { expectedLabels := utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ - druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelAppNameKey: leaseName, }) return MatchFields(IgnoreExtras, Fields{ @@ -294,7 +294,7 @@ func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coor return doGetLatestLeases(cl, etcd, utils.MergeMaps(map[string]string{ - druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, }, etcd.GetDefaultLabels())) } diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 3d0ce4a3a..b5c8b51e7 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -96,9 +96,9 @@ func newStsBuilder(client client.Client, etcdBackupRestoreImage: etcdBackupRestoreImage, initContainerImage: initContainerImage, sts: sts, - clientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, common.DefaultClientPort), - serverPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultServerPort), - backupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, common.DefaultBackupPort), + clientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient), + serverPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer), + backupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore), }, nil } @@ -119,7 +119,7 @@ func (b *stsBuilder) createStatefulSetObjectMeta() { func (b *stsBuilder) getStatefulSetLabels() map[string]string { stsLabels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameStatefulSet, druidv1alpha1.LabelAppNameKey: b.etcd.Name, } return utils.MergeMaps(b.etcd.GetDefaultLabels(), stsLabels) @@ -180,9 +180,9 @@ func (b *stsBuilder) getHostAliases() []corev1.HostAlias { } func (b *stsBuilder) getPodTemplateAnnotations(ctx component.OperatorContext) map[string]string { - if configMapCheckSum, ok := ctx.Data[common.ConfigMapCheckSumKey]; ok { + if configMapCheckSum, ok := ctx.Data[common.CheckSumKeyConfigMap]; ok { return utils.MergeMaps(b.etcd.Spec.Annotations, map[string]string{ - common.ConfigMapCheckSumKey: configMapCheckSum, + common.CheckSumKeyConfigMap: configMapCheckSum, }) } return b.etcd.Spec.Annotations @@ -215,11 +215,11 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { return initContainers } initContainers = append(initContainers, corev1.Container{ - Name: common.ChangePermissionsInitContainerName, + Name: common.InitContainerNameChangePermissions, Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, - Args: []string{fmt.Sprintf("chown -R %d:%d %s", nonRootUser, nonRootUser, common.EtcdDataVolumeMountPath)}, + Args: []string{fmt.Sprintf("chown -R %d:%d %s", nonRootUser, nonRootUser, common.VolumeMountPathEtcdData)}, VolumeMounts: []corev1.VolumeMount{b.getEtcdDataVolumeMount()}, SecurityContext: &corev1.SecurityContext{ RunAsGroup: pointer.Int64(0), @@ -232,7 +232,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() if etcdBackupVolumeMount != nil { initContainers = append(initContainers, corev1.Container{ - Name: common.ChangeBackupBucketPermissionsInitContainerName, + Name: common.InitContainerNameChangeBackupBucketPermissions, Image: b.initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "--"}, @@ -262,7 +262,7 @@ func (b *stsBuilder) getBackupRestoreContainerVolumeMounts() []corev1.VolumeMoun brVolumeMounts = append(brVolumeMounts, b.getEtcdDataVolumeMount(), corev1.VolumeMount{ - Name: common.EtcdConfigVolumeName, + Name: common.VolumeNameEtcdConfig, MountPath: etcdConfigFileMountPath, }, ) @@ -282,20 +282,20 @@ func (b *stsBuilder) getBackupRestoreContainerSecretVolumeMounts() []corev1.Volu if b.etcd.Spec.Backup.TLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: common.BackupRestoreServerTLSVolumeName, - MountPath: common.BackupRestoreServerTLSVolumeMountPath, + Name: common.VolumeNameBackupRestoreServerTLS, + MountPath: common.VolumeMountPathBackupRestoreServerTLS, }, ) } if b.etcd.Spec.Etcd.ClientUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: common.EtcdCAVolumeName, - MountPath: common.EtcdCAVolumeMountPath, + Name: common.VolumeNameEtcdCA, + MountPath: common.VolumeMountPathEtcdCA, }, corev1.VolumeMount{ - Name: common.EtcdClientTLSVolumeName, - MountPath: common.EtcdClientTLSVolumeMountPath, + Name: common.VolumeNameEtcdClientTLS, + MountPath: common.VolumeMountPathEtcdClientTLS, }, ) } @@ -309,25 +309,25 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { if b.etcd.Spec.Backup.Store.Container != nil { if b.useEtcdWrapper { return &corev1.VolumeMount{ - Name: common.LocalBackupVolumeName, + Name: common.VolumeNameLocalBackup, MountPath: fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, "")), } } else { return &corev1.VolumeMount{ - Name: common.LocalBackupVolumeName, + Name: common.VolumeNameLocalBackup, MountPath: pointer.StringDeref(b.etcd.Spec.Backup.Store.Container, ""), } } } case utils.GCS: return &corev1.VolumeMount{ - Name: common.ProviderBackupSecretVolumeName, - MountPath: common.GCSBackupSecretVolumeMountPath, + Name: common.VolumeNameProviderBackupSecret, + MountPath: common.VolumeMountPathGCSBackupSecret, } case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: return &corev1.VolumeMount{ - Name: common.ProviderBackupSecretVolumeName, - MountPath: common.NonGCSProviderBackupSecretVolumeMountPath, + Name: common.VolumeNameProviderBackupSecret, + MountPath: common.VolumeMountPathNonGCSProviderBackupSecret, } } return nil @@ -337,13 +337,13 @@ func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) return corev1.VolumeMount{ Name: volumeClaimTemplateName, - MountPath: common.EtcdDataVolumeMountPath, + MountPath: common.VolumeMountPathEtcdData, } } func (b *stsBuilder) getEtcdContainer() corev1.Container { return corev1.Container{ - Name: common.EtcdContainerName, + Name: common.ContainerNameEtcd, Image: b.etcdImage, ImagePullPolicy: corev1.PullIfNotPresent, Args: b.getEtcdContainerCommandArgs(), @@ -372,7 +372,7 @@ func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { return corev1.Container{}, err } return corev1.Container{ - Name: common.EtcdBackupRestoreContainerName, + Name: common.ContainerNameEtcdBackupRestore, Image: b.etcdBackupRestoreImage, ImagePullPolicy: corev1.PullIfNotPresent, Args: b.getBackupRestoreContainerCommandArgs(), @@ -427,9 +427,9 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { // ----------------------------------------------------------------------------------------------------------------- if b.etcd.Spec.Etcd.ClientUrlTLS != nil { dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - commandArgs = append(commandArgs, fmt.Sprintf("--cacert=%s/%s", common.EtcdCAVolumeMountPath, dataKey)) - commandArgs = append(commandArgs, fmt.Sprintf("--cert=%s/tls.crt", common.EtcdClientTLSVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--key=%s/tls.key", common.EtcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--cacert=%s/%s", common.VolumeMountPathEtcdCA, dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--cert=%s/tls.crt", common.VolumeMountPathEtcdClientTLS)) + commandArgs = append(commandArgs, fmt.Sprintf("--key=%s/tls.key", common.VolumeMountPathEtcdClientTLS)) commandArgs = append(commandArgs, "--insecure-transport=false") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=false") commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) @@ -441,15 +441,15 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) } if b.etcd.Spec.Backup.TLS != nil { - commandArgs = append(commandArgs, fmt.Sprintf("--server-cert=%s/tls.crt", common.BackupRestoreServerTLSVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--server-key=%s/tls.key", common.BackupRestoreServerTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--server-cert=%s/tls.crt", common.VolumeMountPathBackupRestoreServerTLS)) + commandArgs = append(commandArgs, fmt.Sprintf("--server-key=%s/tls.key", common.VolumeMountPathBackupRestoreServerTLS)) } // Other misc command line args // ----------------------------------------------------------------------------------------------------------------- - commandArgs = append(commandArgs, fmt.Sprintf("--data-dir=%s/new.etcd", common.EtcdDataVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--restoration-temp-snapshots-dir=%s/restoration.temp", common.EtcdDataVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--snapstore-temp-directory=%s/temp", common.EtcdDataVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--data-dir=%s/new.etcd", common.VolumeMountPathEtcdData)) + commandArgs = append(commandArgs, fmt.Sprintf("--restoration-temp-snapshots-dir=%s/restoration.temp", common.VolumeMountPathEtcdData)) + commandArgs = append(commandArgs, fmt.Sprintf("--snapstore-temp-directory=%s/temp", common.VolumeMountPathEtcdData)) commandArgs = append(commandArgs, fmt.Sprintf("--etcd-connection-timeout=%s", defaultEtcdConnectionTimeout)) commandArgs = append(commandArgs, "--enable-member-lease-renewal=true") @@ -563,12 +563,12 @@ func (b *stsBuilder) getEtcdContainerReadinessHandler() corev1.ProbeHandler { } scheme := utils.IfConditionOr[corev1.URIScheme](b.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) path := utils.IfConditionOr[string](multiNodeCluster, "/readyz", "/healthz") - port := utils.IfConditionOr[int](multiNodeCluster, defaultWrapperPort, int(common.DefaultBackupPort)) + port := utils.IfConditionOr[int32](multiNodeCluster, common.DefaultPortEtcdWrapper, common.DefaultPortEtcdBackupRestore) return corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: path, - Port: intstr.FromInt(port), + Port: intstr.FromInt32(port), Scheme: scheme, }, } @@ -579,9 +579,9 @@ func (b *stsBuilder) getEtcdContainerReadinessProbeCommand() []string { cmdBuilder.WriteString("ETCDCTL_API=3 etcdctl") if b.etcd.Spec.Etcd.ClientUrlTLS != nil { dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - cmdBuilder.WriteString(fmt.Sprintf(" --cacert=%s/%s", common.EtcdCAVolumeMountPath, dataKey)) - cmdBuilder.WriteString(fmt.Sprintf(" --cert=%s/tls.crt", common.EtcdClientTLSVolumeMountPath)) - cmdBuilder.WriteString(fmt.Sprintf(" --key=%s/tls.key", common.EtcdClientTLSVolumeMountPath)) + cmdBuilder.WriteString(fmt.Sprintf(" --cacert=%s/%s", common.VolumeMountPathEtcdCA, dataKey)) + cmdBuilder.WriteString(fmt.Sprintf(" --cert=%s/tls.crt", common.VolumeMountPathEtcdClientTLS)) + cmdBuilder.WriteString(fmt.Sprintf(" --key=%s/tls.key", common.VolumeMountPathEtcdClientTLS)) cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) } else { cmdBuilder.WriteString(fmt.Sprintf(" --endpoints=http://%s-local:%d", b.etcd.Name, b.clientPort)) @@ -602,7 +602,8 @@ func (b *stsBuilder) getEtcdContainerCommandArgs() []string { return []string{} } commandArgs := []string{"start-etcd"} - commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-host-port=%s-local:8080", b.etcd.Name)) + // TODO + commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-host-port=%s-local:%d", b.etcd.Name, common.DefaultPortEtcdBackupRestore)) commandArgs = append(commandArgs, fmt.Sprintf("--etcd-server-name=%s-local", b.etcd.Name)) if b.etcd.Spec.Etcd.ClientUrlTLS == nil { @@ -610,9 +611,9 @@ func (b *stsBuilder) getEtcdContainerCommandArgs() []string { } else { commandArgs = append(commandArgs, "--backup-restore-tls-enabled=true") dataKey := utils.TypeDeref(b.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.DataKey, "ca.crt") - commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=%s/%s", common.BackupRestoreCAVolumeMountPath, dataKey)) - commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-cert-path=%s/tls.crt", common.EtcdClientTLSVolumeMountPath)) - commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-key-path=%s/tls.key", common.EtcdClientTLSVolumeMountPath)) + commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-ca-cert-bundle-path=%s/%s", common.VolumeMountPathBackupRestoreCA, dataKey)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-cert-path=%s/tls.crt", common.VolumeMountPathEtcdClientTLS)) + commandArgs = append(commandArgs, fmt.Sprintf("--etcd-client-key-path=%s/tls.key", common.VolumeMountPathEtcdClientTLS)) } return commandArgs } @@ -648,36 +649,36 @@ func (b *stsBuilder) getEtcdContainerSecretVolumeMounts() []corev1.VolumeMount { if b.etcd.Spec.Etcd.ClientUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: common.EtcdCAVolumeName, - MountPath: common.EtcdCAVolumeMountPath, + Name: common.VolumeNameEtcdCA, + MountPath: common.VolumeMountPathEtcdCA, }, corev1.VolumeMount{ - Name: common.EtcdServerTLSVolumeName, - MountPath: common.EtcdServerTLSVolumeMountPath, + Name: common.VolumeNameEtcdServerTLS, + MountPath: common.VolumeMountPathEtcdServerTLS, }, corev1.VolumeMount{ - Name: common.EtcdClientTLSVolumeName, - MountPath: common.EtcdClientTLSVolumeMountPath, + Name: common.VolumeNameEtcdClientTLS, + MountPath: common.VolumeMountPathEtcdClientTLS, }, ) } if b.etcd.Spec.Etcd.PeerUrlTLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: common.EtcdPeerCAVolumeName, - MountPath: common.EtcdPeerCAVolumeMountPath, + Name: common.VolumeNameEtcdPeerCA, + MountPath: common.VolumeMountPathEtcdPeerCA, }, corev1.VolumeMount{ - Name: common.EtcdPeerServerTLSVolumeName, - MountPath: common.EtcdPeerServerTLSVolumeMountPath, + Name: common.VolumeNameEtcdPeerServerTLS, + MountPath: common.VolumeMountPathEtcdPeerServerTLS, }, ) } if b.etcd.Spec.Backup.TLS != nil { secretVolumeMounts = append(secretVolumeMounts, corev1.VolumeMount{ - Name: common.BackupRestoreCAVolumeName, - MountPath: common.BackupRestoreCAVolumeMountPath, + Name: common.VolumeNameBackupRestoreCA, + MountPath: common.VolumeMountPathBackupRestoreCA, }, ) } @@ -688,7 +689,7 @@ func (b *stsBuilder) getEtcdContainerSecretVolumeMounts() []corev1.VolumeMount { func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volume, error) { volumes := []corev1.Volume{ { - Name: common.EtcdConfigVolumeName, + Name: common.VolumeNameEtcdConfig, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ @@ -700,7 +701,7 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu Path: etcdConfigFileName, }, }, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -731,29 +732,29 @@ func (b *stsBuilder) getClientTLSVolumes() []corev1.Volume { clientTLSConfig := b.etcd.Spec.Etcd.ClientUrlTLS return []corev1.Volume{ { - Name: common.EtcdCAVolumeName, + Name: common.VolumeNameEtcdCA, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, { - Name: common.EtcdServerTLSVolumeName, + Name: common.VolumeNameEtcdServerTLS, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, { - Name: common.EtcdClientTLSVolumeName, + Name: common.VolumeNameEtcdClientTLS, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: clientTLSConfig.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -764,20 +765,20 @@ func (b *stsBuilder) getPeerTLSVolumes() []corev1.Volume { peerTLSConfig := b.etcd.Spec.Etcd.PeerUrlTLS return []corev1.Volume{ { - Name: common.EtcdPeerCAVolumeName, + Name: common.VolumeNameEtcdPeerCA, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, { - Name: common.EtcdPeerServerTLSVolumeName, + Name: common.VolumeNameEtcdPeerServerTLS, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: peerTLSConfig.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -788,29 +789,29 @@ func (b *stsBuilder) getBackupRestoreTLSVolumes() []corev1.Volume { tlsConfig := b.etcd.Spec.Backup.TLS return []corev1.Volume{ { - Name: common.BackupRestoreCAVolumeName, + Name: common.VolumeNameBackupRestoreCA, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, { - Name: common.BackupRestoreServerTLSVolumeName, + Name: common.VolumeNameBackupRestoreServerTLS, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, { - Name: common.BackupRestoreClientTLSVolumeName, + Name: common.VolumeNameBackupRestoreClientTLS, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tlsConfig.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -831,7 +832,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol hpt := corev1.HostPathDirectory return &corev1.Volume{ - Name: common.LocalBackupVolumeName, + Name: common.VolumeNameLocalBackup, VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: hostPath + "/" + pointer.StringDeref(store.Container, ""), @@ -845,11 +846,11 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol } return &corev1.Volume{ - Name: common.ProviderBackupSecretVolumeName, + Name: common.VolumeNameProviderBackupSecret, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: store.SecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, nil diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index bbf21db47..2c077755f 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -182,10 +182,10 @@ func isStatefulSetPatchedWithPeerTLSVolMount(existingSts *appsv1.StatefulSet) bo volumes := existingSts.Spec.Template.Spec.Volumes var peerURLCAEtcdVolPresent, peerURLEtcdServerTLSVolPresent bool for _, vol := range volumes { - if vol.Name == common.EtcdPeerCAVolumeName { + if vol.Name == common.VolumeNameEtcdPeerCA { peerURLCAEtcdVolPresent = true } - if vol.Name == common.EtcdPeerServerTLSVolumeName { + if vol.Name == common.VolumeNameEtcdPeerServerTLS { peerURLEtcdServerTLSVolPresent = true } } diff --git a/internal/operator/statefulset/statefulset_test.go b/internal/operator/statefulset/statefulset_test.go index 62845c91d..80aaf533f 100644 --- a/internal/operator/statefulset/statefulset_test.go +++ b/internal/operator/statefulset/statefulset_test.go @@ -125,7 +125,7 @@ func TestSyncWhenNoSTSExists(t *testing.T) { }) // *************** Test and assert *************** opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - opCtx.Data[common.ConfigMapCheckSumKey] = testutils.TestConfigMapCheckSum + opCtx.Data[common.CheckSumKeyConfigMap] = testutils.TestConfigMapCheckSum syncErr := operator.Sync(opCtx, etcd) latestSTS, getErr := getLatestStatefulSet(cl, etcd) if tc.expectedErr != nil { diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index 3137b3296..dcde1fa14 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -166,7 +166,7 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { } initContainerMatcherElements := make(map[string]gomegatypes.GomegaMatcher) changePermissionsInitContainerMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ChangePermissionsInitContainerName), + "Name": Equal(common.InitContainerNameChangePermissions), "Image": Equal(s.initContainerImage), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Command": HaveExactElements("sh", "-c", "--"), @@ -174,10 +174,10 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { "VolumeMounts": ConsistOf(s.matchEtcdDataVolMount()), "SecurityContext": matchPodInitContainerSecurityContext(), }) - initContainerMatcherElements[common.ChangePermissionsInitContainerName] = changePermissionsInitContainerMatcher + initContainerMatcherElements[common.InitContainerNameChangePermissions] = changePermissionsInitContainerMatcher if s.etcd.IsBackupStoreEnabled() && s.provider != nil && *s.provider == utils.Local { changeBackupBucketPermissionsMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ChangeBackupBucketPermissionsInitContainerName), + "Name": Equal(common.InitContainerNameChangeBackupBucketPermissions), "Image": Equal(s.initContainerImage), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Command": HaveExactElements("sh", "-c", "--"), @@ -185,15 +185,15 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { "VolumeMounts": ConsistOf(s.getEtcdBackupVolumeMountMatcher()), "SecurityContext": matchPodInitContainerSecurityContext(), }) - initContainerMatcherElements[common.ChangeBackupBucketPermissionsInitContainerName] = changeBackupBucketPermissionsMatcher + initContainerMatcherElements[common.InitContainerNameChangeBackupBucketPermissions] = changeBackupBucketPermissionsMatcher } return MatchAllElements(containerIdentifier, initContainerMatcherElements) } func (s StatefulSetMatcher) matchContainers() gomegatypes.GomegaMatcher { return MatchAllElements(containerIdentifier, Elements{ - common.EtcdContainerName: s.matchEtcdContainer(), - common.EtcdBackupRestoreContainerName: s.matchBackupRestoreContainer(), + common.ContainerNameEtcd: s.matchEtcdContainer(), + common.ContainerNameEtcdBackupRestore: s.matchBackupRestoreContainer(), }) } @@ -320,13 +320,13 @@ func (s StatefulSetMatcher) matchEtcdContainerCmdArgs() gomegatypes.GomegaMatche func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) - return matchVolMount(volumeClaimTemplateName, common.EtcdDataVolumeMountPath) + return matchVolMount(volumeClaimTemplateName, common.VolumeMountPathEtcdData) } func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.GomegaMatcher { volMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 6) volMountMatchers = append(volMountMatchers, s.matchEtcdDataVolMount()) - volMountMatchers = append(volMountMatchers, matchVolMount(common.EtcdConfigVolumeName, etcdConfigFileMountPath)) + volMountMatchers = append(volMountMatchers, matchVolMount(common.VolumeNameEtcdConfig, etcdConfigFileMountPath)) volMountMatchers = append(volMountMatchers, s.getBackupRestoreSecretVolMountMatchers()...) if s.etcd.IsBackupStoreEnabled() { etcdBackupVolMountMatcher := s.getEtcdBackupVolumeMountMatcher() @@ -340,9 +340,9 @@ func (s StatefulSetMatcher) matchBackupRestoreContainerVolMounts() gomegatypes.G func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 3) if s.etcd.Spec.Backup.TLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.BackupRestoreCAVolumeName, common.BackupRestoreCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.BackupRestoreServerTLSVolumeName, common.BackupRestoreServerTLSVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.BackupRestoreClientTLSVolumeName, common.BackupRestoreClientTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameBackupRestoreCA, common.VolumeMountPathBackupRestoreCA)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameBackupRestoreServerTLS, common.VolumeMountPathBackupRestoreServerTLS)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameBackupRestoreClientTLS, common.VolumeMountPathBackupRestoreClientTLS)) } return secretVolMountMatchers } @@ -350,13 +350,13 @@ func (s StatefulSetMatcher) getBackupRestoreSecretVolMountMatchers() []gomegatyp func (s StatefulSetMatcher) getEtcdSecretVolMountsMatchers() []gomegatypes.GomegaMatcher { secretVolMountMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdCAVolumeName, common.EtcdCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdServerTLSVolumeName, common.EtcdServerTLSVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdClientTLSVolumeName, common.EtcdClientTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameEtcdCA, common.VolumeMountPathEtcdCA)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameEtcdServerTLS, common.VolumeMountPathEtcdServerTLS)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameEtcdClientTLS, common.VolumeMountPathEtcdClientTLS)) } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdPeerCAVolumeName, common.EtcdPeerCAVolumeMountPath)) - secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.EtcdPeerServerTLSVolumeName, common.EtcdPeerServerTLSVolumeMountPath)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameEtcdPeerCA, common.VolumeMountPathEtcdPeerCA)) + secretVolMountMatchers = append(secretVolMountMatchers, matchVolMount(common.VolumeNameEtcdPeerServerTLS, common.VolumeMountPathEtcdPeerServerTLS)) } return secretVolMountMatchers } @@ -366,15 +366,15 @@ func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.Gomega case utils.Local: if s.etcd.Spec.Backup.Store.Container != nil { if s.useEtcdWrapper { - return matchVolMount(common.LocalBackupVolumeName, fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))) + return matchVolMount(common.VolumeNameLocalBackup, fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))) } else { - return matchVolMount(common.LocalBackupVolumeName, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) + return matchVolMount(common.VolumeNameLocalBackup, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) } } case utils.GCS: - return matchVolMount(common.ProviderBackupSecretVolumeName, common.GCSBackupSecretVolumeMountPath) + return matchVolMount(common.VolumeNameProviderBackupSecret, common.VolumeMountPathGCSBackupSecret) case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: - return matchVolMount(common.ProviderBackupSecretVolumeName, common.NonGCSProviderBackupSecretVolumeMountPath) + return matchVolMount(common.VolumeNameProviderBackupSecret, common.VolumeMountPathNonGCSProviderBackupSecret) } return nil } @@ -411,7 +411,7 @@ func (s StatefulSetMatcher) matchEtcdPodSecurityContext() gomegatypes.GomegaMatc func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { volMatchers := make([]gomegatypes.GomegaMatcher, 0, 7) etcdConfigFileVolMountMatcher := MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EtcdConfigVolumeName), + "Name": Equal(common.VolumeNameEtcdConfig), "VolumeSource": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ @@ -421,7 +421,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { "Key": Equal(etcdConfigFileName), "Path": Equal(etcdConfigFileName), })), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), }) @@ -443,81 +443,81 @@ func (s StatefulSetMatcher) getPodSecurityVolumeMatchers() []gomegatypes.GomegaM volMatchers := make([]gomegatypes.GomegaMatcher, 0, 5) if s.etcd.Spec.Etcd.ClientUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EtcdCAVolumeName), + "Name": Equal(common.VolumeNameEtcdCA), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EtcdServerTLSVolumeName), + "Name": Equal(common.VolumeNameEtcdServerTLS), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EtcdClientTLSVolumeName), + "Name": Equal(common.VolumeNameEtcdClientTLS), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) } if s.etcd.Spec.Etcd.PeerUrlTLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EtcdPeerCAVolumeName), + "Name": Equal(common.VolumeNameEtcdPeerCA), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.EtcdPeerServerTLSVolumeName), + "Name": Equal(common.VolumeNameEtcdPeerServerTLS), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) } if s.etcd.Spec.Backup.TLS != nil { volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.BackupRestoreCAVolumeName), + "Name": Equal(common.VolumeNameBackupRestoreCA), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.TLSCASecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.BackupRestoreServerTLSVolumeName), + "Name": Equal(common.VolumeNameBackupRestoreServerTLS), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ServerTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) volMatchers = append(volMatchers, MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.BackupRestoreClientTLSVolumeName), + "Name": Equal(common.VolumeNameBackupRestoreClientTLS), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.TLS.ClientTLSSecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), })) @@ -534,7 +534,7 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { hostPath, err := utils.GetHostMountPathFromSecretRef(context.Background(), s.cl, logr.Discard(), s.etcd.Spec.Backup.Store, s.etcd.Namespace) s.g.Expect(err).ToNot(HaveOccurred()) return MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.LocalBackupVolumeName), + "Name": Equal(common.VolumeNameLocalBackup), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "HostPath": PointTo(MatchFields(IgnoreExtras, Fields{ "Path": Equal(fmt.Sprintf("%s/%s", hostPath, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))), @@ -546,11 +546,11 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: s.g.Expect(s.etcd.Spec.Backup.Store.SecretRef).ToNot(BeNil()) return MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), + "Name": Equal(common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(s.etcd.Spec.Backup.Store.SecretRef.Name), - "DefaultMode": PointTo(Equal(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), }), }) @@ -575,7 +575,7 @@ func matchPodInitContainerSecurityContext() gomegatypes.GomegaMatcher { func getStatefulSetLabels(etcdName string) map[string]string { return map[string]string{ - druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameStatefulSet, druidv1alpha1.LabelAppNameKey: etcdName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: etcdName, diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index 935c651db..6b1d7f134 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -61,20 +61,20 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) switch provider { case S3: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) case ABS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) case GCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.GCSBackupSecretVolumeMountPath))) + envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.VolumeMountPathGCSBackupSecret))) envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) case Swift: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) case OSS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) case ECS: if store.SecretRef == nil { @@ -85,7 +85,7 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) case OCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.NonGCSProviderBackupSecretVolumeMountPath)) + envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) } return envVars, nil diff --git a/internal/utils/image.go b/internal/utils/image.go index 18f1cfc81..c56c1ae2f 100755 --- a/internal/utils/image.go +++ b/internal/utils/image.go @@ -35,13 +35,13 @@ func GetEtcdImages(etcd *druidv1alpha1.Etcd, iv imagevector.ImageVector, useEtcd } func getEtcdImageKeys(useEtcdWrapper bool) (etcdImageKey string, etcdBRImageKey string, alpine string) { - alpine = common.Alpine + alpine = common.ImageKeyAlpine if useEtcdWrapper { - etcdImageKey = common.EtcdWrapper - etcdBRImageKey = common.BackupRestoreDistroless + etcdImageKey = common.ImageKeyEtcdWrapper + etcdBRImageKey = common.ImageKeyEtcdBackupRestoreDistroless } else { - etcdImageKey = common.Etcd - etcdBRImageKey = common.BackupRestore + etcdImageKey = common.ImageKeyEtcd + etcdBRImageKey = common.ImageKeyEtcdBackupRestore } return } @@ -68,5 +68,5 @@ func GetEtcdBackupRestoreImage(iv imagevector.ImageVector, useEtcdWrapper bool) // GetInitContainerImage returns the image for init container from the given image vector. func GetInitContainerImage(iv imagevector.ImageVector) (*string, error) { - return chooseImage(common.Alpine, nil, iv) + return chooseImage(common.ImageKeyAlpine, nil, iv) } diff --git a/internal/utils/image_test.go b/internal/utils/image_test.go index ad4542c1b..0d16fae43 100644 --- a/internal/utils/image_test.go +++ b/internal/utils/image_test.go @@ -44,7 +44,7 @@ func testWithEtcdAndEtcdBRImages(g *WithT, etcd *druidv1alpha1.Etcd) { g.Expect(etcdImg).To(Equal(*etcd.Spec.Etcd.Image)) g.Expect(etcdBRImg).ToNot(BeEmpty()) g.Expect(etcdBRImg).To(Equal(*etcd.Spec.Backup.Image)) - vectorInitContainerImage, err := iv.FindImage(common.Alpine) + vectorInitContainerImage, err := iv.FindImage(common.ImageKeyAlpine) g.Expect(err).To(BeNil()) g.Expect(initContainerImg).To(Equal(vectorInitContainerImage.String())) } @@ -56,14 +56,14 @@ func testWithNoImageInSpecAndIVWithEtcdAndBRImages(g *WithT, etcd *druidv1alpha1 etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) g.Expect(err).To(BeNil()) g.Expect(etcdImage).ToNot(BeEmpty()) - vectorEtcdImage, err := iv.FindImage(common.Etcd) + vectorEtcdImage, err := iv.FindImage(common.ImageKeyEtcd) g.Expect(err).To(BeNil()) g.Expect(etcdImage).To(Equal(vectorEtcdImage.String())) g.Expect(etcdBackupRestoreImage).ToNot(BeNil()) - vectorBackupRestoreImage, err := iv.FindImage(common.BackupRestore) + vectorBackupRestoreImage, err := iv.FindImage(common.ImageKeyEtcdBackupRestore) g.Expect(err).To(BeNil()) g.Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) - vectorInitContainerImage, err := iv.FindImage(common.Alpine) + vectorInitContainerImage, err := iv.FindImage(common.ImageKeyAlpine) g.Expect(err).To(BeNil()) g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) } @@ -74,12 +74,12 @@ func testSpecWithEtcdBRImageAndIVWithEtcdImage(g *WithT, etcd *druidv1alpha1.Etc etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, false) g.Expect(err).To(BeNil()) g.Expect(etcdImage).ToNot(BeEmpty()) - vectorEtcdImage, err := iv.FindImage(common.Etcd) + vectorEtcdImage, err := iv.FindImage(common.ImageKeyEtcd) g.Expect(err).To(BeNil()) g.Expect(etcdImage).To(Equal(vectorEtcdImage.String())) g.Expect(etcdBackupRestoreImage).ToNot(BeNil()) g.Expect(etcdBackupRestoreImage).To(Equal(*etcd.Spec.Backup.Image)) - vectorInitContainerImage, err := iv.FindImage(common.Alpine) + vectorInitContainerImage, err := iv.FindImage(common.ImageKeyAlpine) g.Expect(err).To(BeNil()) g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) } @@ -111,14 +111,14 @@ func testWithNoImagesInSpecAndIVWithAllImagesWithWrapper(g *WithT, etcd *druidv1 etcdImage, etcdBackupRestoreImage, initContainerImage, err := GetEtcdImages(etcd, iv, true) g.Expect(err).To(BeNil()) g.Expect(etcdImage).ToNot(BeEmpty()) - vectorEtcdImage, err := iv.FindImage(common.EtcdWrapper) + vectorEtcdImage, err := iv.FindImage(common.ImageKeyEtcdWrapper) g.Expect(err).To(BeNil()) g.Expect(etcdImage).To(Equal(vectorEtcdImage.String())) g.Expect(etcdBackupRestoreImage).ToNot(BeEmpty()) - vectorBackupRestoreImage, err := iv.FindImage(common.BackupRestoreDistroless) + vectorBackupRestoreImage, err := iv.FindImage(common.ImageKeyEtcdBackupRestoreDistroless) g.Expect(err).To(BeNil()) g.Expect(etcdBackupRestoreImage).To(Equal(vectorBackupRestoreImage.String())) - vectorInitContainerImage, err := iv.FindImage(common.Alpine) + vectorInitContainerImage, err := iv.FindImage(common.ImageKeyAlpine) g.Expect(err).To(BeNil()) g.Expect(initContainerImage).To(Equal(vectorInitContainerImage.String())) } diff --git a/internal/utils/lease.go b/internal/utils/lease.go index ef766d4a4..ff1bed0b6 100644 --- a/internal/utils/lease.go +++ b/internal/utils/lease.go @@ -23,7 +23,7 @@ const peerURLTLSEnabledKey = "member.etcd.gardener.cloud/tls-enabled" func IsPeerURLTLSEnabledForMembers(ctx context.Context, cl client.Client, logger logr.Logger, namespace, etcdName string, numReplicas int32) (bool, error) { leaseList := &coordinationv1.LeaseList{} if err := cl.List(ctx, leaseList, client.InNamespace(namespace), client.MatchingLabels(map[string]string{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelPartOfKey: etcdName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, })); err != nil { diff --git a/internal/utils/lease_test.go b/internal/utils/lease_test.go index 082612637..8440c3ed5 100644 --- a/internal/utils/lease_test.go +++ b/internal/utils/lease_test.go @@ -67,7 +67,7 @@ func TestIsPeerURLTLSEnabledForAllMembers(t *testing.T) { existingObjects = append(existingObjects, l) } cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, testutils.TestNamespace, map[string]string{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelPartOfKey: testutils.TestEtcdName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, }, existingObjects...) @@ -84,7 +84,7 @@ func TestIsPeerURLTLSEnabledForAllMembers(t *testing.T) { func createLeases(namespace, etcdName string, numLease, withTLSEnabled int) []*coordinationv1.Lease { leases := make([]*coordinationv1.Lease, 0, numLease) labels := map[string]string{ - druidv1alpha1.LabelComponentKey: common.MemberLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelPartOfKey: etcdName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, } diff --git a/test/e2e/utils.go b/test/e2e/utils.go index d918bcd2e..1a1cf71cc 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -840,7 +840,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -849,7 +849,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -858,7 +858,7 @@ func etcdZeroDownTimeValidatorJob(etcdSvc, testName string, tls *v1alpha1.TLSCon VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tls.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -972,7 +972,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -981,7 +981,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, @@ -990,7 +990,7 @@ func getDebugPod(etcd *v1alpha1.Etcd) *corev1.Pod { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name, - DefaultMode: pointer.Int32(common.OwnerReadWriteGroupReadPermissions), + DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), }, }, }, diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index 5ee6164d3..2c6c960b9 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -389,9 +389,9 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container): Equal(fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container)), }), "VolumeMounts": MatchElements(testutils.VolumeMountIterator, IgnoreExtras, Elements{ - common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), - "MountPath": Equal(common.GCSBackupSecretVolumeMountPath), + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.VolumeNameProviderBackupSecret), + "MountPath": Equal(common.VolumeMountPathGCSBackupSecret), }), }), "Env": MatchAllElements(testutils.EnvIterator, Elements{ @@ -435,8 +435,8 @@ func validateStoreGCPForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -485,14 +485,14 @@ func validateStoreAWSForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.J }), common.EnvAWSApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAWSApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), + "Value": Equal(common.VolumeMountPathNonGCSProviderBackupSecret), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -541,14 +541,14 @@ func validateStoreAzureForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1 }), common.EnvAzureApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAzureApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), + "Value": Equal(common.VolumeMountPathNonGCSProviderBackupSecret), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -597,14 +597,14 @@ func validateStoreOpenstackForCompactionJob(instance *druidv1alpha1.Etcd, j *bat }), common.EnvOpenstackApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvOpenstackApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), + "Value": Equal(common.VolumeMountPathNonGCSProviderBackupSecret), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), @@ -654,14 +654,14 @@ func validateStoreAlicloudForCompactionJob(instance *druidv1alpha1.Etcd, j *batc }), common.EnvAlicloudApplicationCredentials: MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.EnvAlicloudApplicationCredentials), - "Value": Equal(common.NonGCSProviderBackupSecretVolumeMountPath), + "Value": Equal(common.VolumeMountPathNonGCSProviderBackupSecret), }), }), }), }), "Volumes": MatchElements(testutils.VolumeIterator, IgnoreExtras, Elements{ - common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(common.ProviderBackupSecretVolumeName), + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(instance.Spec.Backup.Store.SecretRef.Name), diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index ba20c59b6..1631db245 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -127,16 +127,16 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I targetProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) Expect(err).NotTo(HaveOccurred()) - images, err := imagevector.FindImages(imageVector, []string{common.BackupRestore}) + images, err := imagevector.FindImages(imageVector, []string{common.ImageKeyEtcdBackupRestore}) Expect(err).NotTo(HaveOccurred()) - backupRestoreImage := images[common.BackupRestore] + backupRestoreImage := images[common.ImageKeyEtcdBackupRestore] matcher := MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), @@ -156,7 +156,7 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Template": MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.EtcdCopyBackupTaskComponentName), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), @@ -334,15 +334,15 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { switch storeProvider { case "GCS": return Elements{ - volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), + volumePrefix + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.VolumeNameProviderBackupSecret), "MountPath": Equal(fmt.Sprintf("/var/.%sgcp/", volumePrefix)), }), } default: return Elements{ - volumePrefix + common.ProviderBackupSecretVolumeName: MatchFields(IgnoreExtras, Fields{ - "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), + volumePrefix + common.VolumeNameProviderBackupSecret: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(volumePrefix + common.VolumeNameProviderBackupSecret), "MountPath": Equal(fmt.Sprintf("/var/%setcd-backup", volumePrefix)), }), } @@ -352,12 +352,12 @@ func getVolumeMountsElements(storeProvider, volumePrefix string) Elements { func getVolumesElements(volumePrefix string, store *druidv1alpha1.StoreSpec) Elements { return Elements{ - volumePrefix + common.ProviderBackupSecretVolumeName: MatchAllFields(Fields{ - "Name": Equal(volumePrefix + common.ProviderBackupSecretVolumeName), + volumePrefix + common.VolumeNameProviderBackupSecret: MatchAllFields(Fields{ + "Name": Equal(volumePrefix + common.VolumeNameProviderBackupSecret), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ "SecretName": Equal(store.SecretRef.Name), - "DefaultMode": Equal(pointer.Int32(common.OwnerReadWriteGroupReadPermissions)), + "DefaultMode": Equal(pointer.Int32(common.ModeOwnerReadWriteGroupRead)), })), }), }), diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 2409f3e70..6089b6e20 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -260,7 +260,7 @@ func testPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi testClientBuilder := testutils.NewTestClientBuilder(). RecordErrorForObjects(testutils.ClientMethodDelete, testutils.TestAPIInternalErr, client.ObjectKey{Name: etcdInstance.GetClientServiceName(), Namespace: etcdInstance.Namespace}). RecordErrorForObjectsMatchingLabels(testutils.ClientMethodDeleteAll, etcdInstance.Namespace, map[string]string{ - druidv1alpha1.LabelComponentKey: common.SnapshotLeaseComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: etcdInstance.Name, }, testutils.TestAPIInternalErr) diff --git a/test/utils/etcd.go b/test/utils/etcd.go index 9d449a7ce..a6f204f01 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -397,8 +397,8 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { "memory": ParseQuantity("1000Mi"), }, }, - ClientPort: pointer.Int32(common.DefaultClientPort), - ServerPort: pointer.Int32(common.DefaultServerPort), + ClientPort: pointer.Int32(common.DefaultPortEtcdClient), + ServerPort: pointer.Int32(common.DefaultPortEtcdPeer), }, Common: druidv1alpha1.SharedConfig{ AutoCompactionMode: &autoCompactionMode, @@ -411,7 +411,7 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { func getBackupSpec() druidv1alpha1.BackupSpec { return druidv1alpha1.BackupSpec{ Image: &imageBR, - Port: pointer.Int32(common.DefaultBackupPort), + Port: pointer.Int32(common.DefaultPortEtcdBackupRestore), FullSnapshotSchedule: &snapshotSchedule, GarbageCollectionPolicy: &garbageCollectionPolicy, GarbageCollectionPeriod: &garbageCollectionPeriod, diff --git a/test/utils/etcdcopybackupstask.go b/test/utils/etcdcopybackupstask.go index f2e350c23..2bcb7deb7 100644 --- a/test/utils/etcdcopybackupstask.go +++ b/test/utils/etcdcopybackupstask.go @@ -110,7 +110,7 @@ func getLabels(taskName, jobName string, includeNetworkPolicyLabels bool) map[st labels := map[string]string{ druidv1alpha1.LabelPartOfKey: taskName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, - druidv1alpha1.LabelComponentKey: common.EtcdCopyBackupTaskComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameEtcdCopyBackupsTask, druidv1alpha1.LabelAppNameKey: jobName, } if includeNetworkPolicyLabels { diff --git a/test/utils/imagevector.go b/test/utils/imagevector.go index 5a9dd73ed..ab2c222fb 100644 --- a/test/utils/imagevector.go +++ b/test/utils/imagevector.go @@ -15,14 +15,14 @@ func CreateImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperIma var imageSources []*imagevector.ImageSource if withEtcdImage { imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.Etcd, + Name: common.ImageKeyEtcd, Repository: TestImageRepo, Tag: pointer.String(ETCDImageSourceTag), }) } if withBackupRestoreImage { imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.BackupRestore, + Name: common.ImageKeyEtcdBackupRestore, Repository: TestImageRepo, Tag: pointer.String(ETCDBRImageTag), }) @@ -30,20 +30,20 @@ func CreateImageVector(withEtcdImage, withBackupRestoreImage, withEtcdWrapperIma } if withEtcdWrapperImage { imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.EtcdWrapper, + Name: common.ImageKeyEtcdWrapper, Repository: TestImageRepo, Tag: pointer.String(ETCDWrapperImageTag), }) } if withBackupRestoreDistrolessImage { imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.BackupRestoreDistroless, + Name: common.ImageKeyEtcdBackupRestoreDistroless, Repository: TestImageRepo, Tag: pointer.String(ETCDBRDistrolessImageTag), }) } imageSources = append(imageSources, &imagevector.ImageSource{ - Name: common.Alpine, + Name: common.ImageKeyAlpine, Repository: TestImageRepo, Tag: pointer.String(InitContainerTag), }) diff --git a/test/utils/statefulset.go b/test/utils/statefulset.go index d24962b17..c292b8474 100644 --- a/test/utils/statefulset.go +++ b/test/utils/statefulset.go @@ -52,7 +52,7 @@ func CreateStatefulSet(name, namespace string, etcdUID types.UID, replicas int32 Labels: map[string]string{ druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: name, - druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameStatefulSet, druidv1alpha1.LabelAppNameKey: name, }, Annotations: nil, @@ -80,7 +80,7 @@ func CreateStatefulSet(name, namespace string, etcdUID types.UID, replicas int32 Labels: map[string]string{ druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: name, - druidv1alpha1.LabelComponentKey: common.StatefulSetComponentName, + druidv1alpha1.LabelComponentKey: common.ComponentNameStatefulSet, druidv1alpha1.LabelAppNameKey: name, }, }, From 09431d88a838bd53f718d9738d4cf2391341d9d4 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 9 May 2024 12:30:59 +0530 Subject: [PATCH 164/235] Address review comments by @renormalize: sentinel webhook disabled by default in helm charts, but enabled via skaffold --- charts/druid/values.yaml | 2 +- internal/operator/statefulset/builder.go | 4 ++-- internal/operator/statefulset/statefulset.go | 2 ++ internal/utils/store.go | 4 ++-- skaffold.yaml | 3 +++ 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index ff57cb6de..21c93ee70 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -50,7 +50,7 @@ controllers: webhooks: sentinel: - enabled: true + enabled: false # reconciler-service-account: system:serviceaccount:{{ .Release.Namespace }}:etcd-druid exemptServiceAccounts: - system:serviceaccount:kube-system:generic-garbage-collector diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index b5c8b51e7..8df4dcfb2 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -697,8 +697,8 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu }, Items: []corev1.KeyToPath{ { - Key: etcdConfigFileName, - Path: etcdConfigFileName, + Key: etcdConfigFileName, // etcd.conf.yaml key in the configmap, which contains the config for the etcd as yaml + Path: etcdConfigFileName, // sub-path under the volume mount path, to store the contents of configmap key etcd.conf.yaml }, }, DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 2c077755f..7f1335dcb 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -114,6 +114,8 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } +// getExistingStatefulSet gets the existing statefulset if it exists. +// If it is not found, it simply returns nil. Any other errors are returned as is. func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { sts := emptyStatefulSet(etcd) if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { diff --git a/internal/utils/store.go b/internal/utils/store.go index fc1eb7695..924bc45f6 100644 --- a/internal/utils/store.go +++ b/internal/utils/store.go @@ -73,7 +73,7 @@ func GetHostMountPathFromSecretRef(ctx context.Context, client client.Client, lo // StorageProviderFromInfraProvider converts infra to object store provider. func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (string, error) { - if infra == nil || len(*infra) == 0 { + if infra == nil { return "", nil } @@ -95,6 +95,6 @@ func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (str case Local, druidv1alpha1.StorageProvider(strings.ToLower(Local)): return Local, nil default: - return "", fmt.Errorf("unsupported storage provider: %v", *infra) + return "", fmt.Errorf("unsupported storage provider: '%v'", *infra) } } diff --git a/skaffold.yaml b/skaffold.yaml index d5c3b1ab1..a4ad90abb 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -27,6 +27,9 @@ deploy: compaction: etcdEventsThreshold: 15 metricsScrapeWaitDuration: 30s + webhooks: + sentinel: + enabled: true profiles: - name: e2e-test activation: From b9066f3b4f1addd64e59a5cf11028a206fc6926b Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 9 May 2024 12:49:05 +0530 Subject: [PATCH 165/235] Address review comments by @renormalize: Don't specify explicit type arguments for generic functions --- .../operator/clientservice/clientservice.go | 6 ++--- .../clientservice/clientservice_test.go | 6 ++--- internal/operator/configmap/configmap_test.go | 16 ++++++------ internal/operator/configmap/etcdconfig.go | 16 ++++++------ internal/operator/peerservice/peerservice.go | 2 +- .../operator/peerservice/peerservice_test.go | 2 +- internal/operator/statefulset/builder.go | 26 +++++++++---------- internal/operator/statefulset/stsmatcher.go | 18 ++++++------- internal/utils/store_test.go | 2 +- test/it/controller/etcd/assertions.go | 10 +++---- test/utils/etcd.go | 8 +++--- 11 files changed, 56 insertions(+), 56 deletions(-) diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index d3b321b89..49376acf0 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -146,9 +146,9 @@ func emptyClientService(objectKey client.ObjectKey) *corev1.Service { } func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + backupPort := utils.TypeDeref(etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) + clientPort := utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) + peerPort := utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) return []corev1.ServicePort{ { diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/operator/clientservice/clientservice_test.go index 49db58bcc..7dc344fd9 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/operator/clientservice/clientservice_test.go @@ -257,9 +257,9 @@ func buildEtcd(clientPort, peerPort, backupPort *int32) *druidv1alpha1.Etcd { } func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { - clientPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) - backupPort := utils.TypeDeref[int32](etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + clientPort := utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) + backupPort := utils.TypeDeref(etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) + peerPort := utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) expectedLabels := etcd.GetDefaultLabels() var expectedAnnotations map[string]string diff --git a/internal/operator/configmap/configmap_test.go b/internal/operator/configmap/configmap_test.go index eb95bc36d..66cc92bff 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/operator/configmap/configmap_test.go @@ -174,7 +174,7 @@ func TestPrepareInitialCluster(t *testing.T) { t.Run(tc.name, func(t *testing.T) { etcd := buildEtcd(tc.etcdReplicas, true, tc.peerTLSEnabled) etcd.Spec.Etcd.ServerPort = tc.etcdSpecServerPort - peerScheme := utils.IfConditionOr[string](etcd.Spec.Etcd.PeerUrlTLS != nil, "https", "http") + peerScheme := utils.IfConditionOr(etcd.Spec.Etcd.PeerUrlTLS != nil, "https", "http") actualInitialCluster := prepareInitialCluster(etcd, peerScheme) g.Expect(actualInitialCluster).To(Equal(tc.expectedInitialCluster)) }) @@ -342,8 +342,8 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C "quota-backend-bytes": Equal(etcd.Spec.Etcd.Quota.Value()), "initial-cluster-token": Equal("etcd-cluster"), "initial-cluster-state": Equal("new"), - "auto-compaction-mode": Equal(string(utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic))), - "auto-compaction-retention": Equal(utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention)), + "auto-compaction-mode": Equal(string(utils.TypeDeref(etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic))), + "auto-compaction-retention": Equal(utils.TypeDeref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention)), })) matchClientTLSRelatedConfiguration(g, etcd, actualETCDConfig) matchPeerTLSRelatedConfiguration(g, etcd, actualETCDConfig) @@ -352,8 +352,8 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -364,7 +364,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("client-transport-security")) } @@ -380,12 +380,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), "auto-tls": Equal(false), }), - "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), + "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), + "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) diff --git a/internal/operator/configmap/etcdconfig.go b/internal/operator/configmap/etcdconfig.go index 38ec83e93..adb810ccc 100644 --- a/internal/operator/configmap/etcdconfig.go +++ b/internal/operator/configmap/etcdconfig.go @@ -73,19 +73,19 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { cfg := &etcdConfig{ Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), DataDir: defaultDataDir, - Metrics: utils.TypeDeref[druidv1alpha1.MetricsLevel](etcd.Spec.Etcd.Metrics, druidv1alpha1.Basic), + Metrics: utils.TypeDeref(etcd.Spec.Etcd.Metrics, druidv1alpha1.Basic), SnapshotCount: defaultSnapshotCount, EnableV2: false, QuotaBackendBytes: getDBQuotaBytes(etcd), InitialClusterToken: defaultInitialClusterToken, InitialClusterState: defaultInitialClusterState, InitialCluster: prepareInitialCluster(etcd, peerScheme), - AutoCompactionMode: utils.TypeDeref[druidv1alpha1.CompactionMode](etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), - AutoCompactionRetention: utils.TypeDeref[string](etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), - ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), - ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), - AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), - AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref[int32](etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), + AutoCompactionMode: utils.TypeDeref(etcd.Spec.Common.AutoCompactionMode, druidv1alpha1.Periodic), + AutoCompactionRetention: utils.TypeDeref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), + ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), + ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), + AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), + AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -112,7 +112,7 @@ func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, caPath, serv CertFile: fmt.Sprintf("%s/tls.crt", serverTLSPath), KeyFile: fmt.Sprintf("%s/tls.key", serverTLSPath), ClientCertAuth: true, - TrustedCAFile: fmt.Sprintf("%s/%s", caPath, utils.TypeDeref[string](tlsConfig.TLSCASecretRef.DataKey, defaultTLSCASecretKey)), + TrustedCAFile: fmt.Sprintf("%s/%s", caPath, utils.TypeDeref(tlsConfig.TLSCASecretRef.DataKey, defaultTLSCASecretKey)), AutoTLS: false, } } diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index 761f7a5e9..e8dfa8bfc 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -132,7 +132,7 @@ func emptyPeerService(objectKey client.ObjectKey) *corev1.Service { } func getPorts(etcd *druidv1alpha1.Etcd) []corev1.ServicePort { - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + peerPort := utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) return []corev1.ServicePort{ { Name: "peer", diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/operator/peerservice/peerservice_test.go index e957df624..7c84ef2b1 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/operator/peerservice/peerservice_test.go @@ -239,7 +239,7 @@ func newPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { } func matchPeerService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { - peerPort := utils.TypeDeref[int32](etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + peerPort := utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(etcd.GetPeerServiceName()), diff --git a/internal/operator/statefulset/builder.go b/internal/operator/statefulset/builder.go index 8df4dcfb2..05445430d 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/operator/statefulset/builder.go @@ -163,7 +163,7 @@ func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error Affinity: b.etcd.Spec.SchedulingConstraints.Affinity, TopologySpreadConstraints: b.etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, Volumes: podVolumes, - PriorityClassName: utils.TypeDeref[string](b.etcd.Spec.PriorityClassName, ""), + PriorityClassName: utils.TypeDeref(b.etcd.Spec.PriorityClassName, ""), }, }, } @@ -192,7 +192,7 @@ func (b *stsBuilder) getVolumeClaimTemplates() []corev1.PersistentVolumeClaim { return []corev1.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ - Name: utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name), + Name: utils.TypeDeref(b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name), }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ @@ -200,7 +200,7 @@ func (b *stsBuilder) getVolumeClaimTemplates() []corev1.PersistentVolumeClaim { }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: utils.TypeDeref[apiresource.Quantity](b.etcd.Spec.StorageCapacity, defaultStorageCapacity), + corev1.ResourceStorage: utils.TypeDeref(b.etcd.Spec.StorageCapacity, defaultStorageCapacity), }, }, StorageClassName: b.etcd.Spec.StorageClass, @@ -334,7 +334,7 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { } func (b *stsBuilder) getEtcdDataVolumeMount() corev1.VolumeMount { - volumeClaimTemplateName := utils.TypeDeref[string](b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) + volumeClaimTemplateName := utils.TypeDeref(b.etcd.Spec.VolumeClaimTemplate, b.etcd.Name) return corev1.VolumeMount{ Name: volumeClaimTemplateName, MountPath: common.VolumeMountPathEtcdData, @@ -360,7 +360,7 @@ func (b *stsBuilder) getEtcdContainer() corev1.Container { ContainerPort: b.clientPort, }, }, - Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Etcd.Resources, defaultResourceRequirements), + Resources: utils.TypeDeref(b.etcd.Spec.Etcd.Resources, defaultResourceRequirements), Env: b.getEtcdContainerEnvVars(), VolumeMounts: b.getEtcdContainerVolumeMounts(), } @@ -384,7 +384,7 @@ func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { }, }, Env: env, - Resources: utils.TypeDeref[corev1.ResourceRequirements](b.etcd.Spec.Backup.Resources, defaultResourceRequirements), + Resources: utils.TypeDeref(b.etcd.Spec.Backup.Resources, defaultResourceRequirements), VolumeMounts: b.getBackupRestoreContainerVolumeMounts(), }, nil } @@ -458,7 +458,7 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { quota = b.etcd.Spec.Etcd.Quota.Value() } commandArgs = append(commandArgs, fmt.Sprintf("--embedded-etcd-quota-bytes=%d", quota)) - if utils.TypeDeref[bool](b.etcd.Spec.Backup.EnableProfiling, false) { + if utils.TypeDeref(b.etcd.Spec.Backup.EnableProfiling, false) { commandArgs = append(commandArgs, "--enable-profiling=true") } @@ -517,7 +517,7 @@ func (b *stsBuilder) getBackupStoreCommandArgs() []string { } commandArgs = append(commandArgs, fmt.Sprintf("--garbage-collection-policy=%s", garbageCollectionPolicy)) if garbageCollectionPolicy == "LimitBased" { - commandArgs = append(commandArgs, fmt.Sprintf("--max-backups=%d", utils.TypeDeref[int32](b.etcd.Spec.Backup.MaxBackupsLimitBasedGC, defaultMaxBackupsLimitBasedGC))) + commandArgs = append(commandArgs, fmt.Sprintf("--max-backups=%d", utils.TypeDeref(b.etcd.Spec.Backup.MaxBackupsLimitBasedGC, defaultMaxBackupsLimitBasedGC))) } if b.etcd.Spec.Backup.GarbageCollectionPeriod != nil { commandArgs = append(commandArgs, fmt.Sprintf("--garbage-collection-period=%s", b.etcd.Spec.Backup.GarbageCollectionPeriod.Duration.String())) @@ -526,7 +526,7 @@ func (b *stsBuilder) getBackupStoreCommandArgs() []string { // Snapshot compression and timeout command line args // ----------------------------------------------------------------------------------------------------------------- if b.etcd.Spec.Backup.SnapshotCompression != nil { - if utils.TypeDeref[bool](b.etcd.Spec.Backup.SnapshotCompression.Enabled, false) { + if utils.TypeDeref(b.etcd.Spec.Backup.SnapshotCompression.Enabled, false) { commandArgs = append(commandArgs, fmt.Sprintf("--compress-snapshots=%t", *b.etcd.Spec.Backup.SnapshotCompression.Enabled)) } if b.etcd.Spec.Backup.SnapshotCompression.Policy != nil { @@ -561,9 +561,9 @@ func (b *stsBuilder) getEtcdContainerReadinessHandler() corev1.ProbeHandler { }, } } - scheme := utils.IfConditionOr[corev1.URIScheme](b.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) - path := utils.IfConditionOr[string](multiNodeCluster, "/readyz", "/healthz") - port := utils.IfConditionOr[int32](multiNodeCluster, common.DefaultPortEtcdWrapper, common.DefaultPortEtcdBackupRestore) + scheme := utils.IfConditionOr(b.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) + path := utils.IfConditionOr(multiNodeCluster, "/readyz", "/healthz") + port := utils.IfConditionOr(multiNodeCluster, common.DefaultPortEtcdWrapper, common.DefaultPortEtcdBackupRestore) return corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -623,7 +623,7 @@ func (b *stsBuilder) getEtcdContainerEnvVars() []corev1.EnvVar { return []corev1.EnvVar{} } backTLSEnabled := b.etcd.Spec.Backup.TLS != nil - scheme := utils.IfConditionOr[string](backTLSEnabled, "https", "http") + scheme := utils.IfConditionOr(backTLSEnabled, "https", "http") endpoint := fmt.Sprintf("%s://%s-local:%d", scheme, b.etcd.Name, b.backupPort) return []corev1.EnvVar{ diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index dcde1fa14..ce700290e 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -109,13 +109,13 @@ func (s StatefulSetMatcher) matchVolumeClaimTemplates() gomegatypes.GomegaMatche defaultStorageCapacity := apiresource.MustParse("16Gi") return ConsistOf(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name)), + "Name": Equal(utils.TypeDeref(s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name)), }), "Spec": MatchFields(IgnoreExtras, Fields{ "AccessModes": ConsistOf(corev1.ReadWriteOnce), "Resources": MatchFields(IgnoreExtras, Fields{ "Requests": MatchKeys(IgnoreExtras, Keys{ - corev1.ResourceStorage: Equal(utils.TypeDeref[apiresource.Quantity](s.etcd.Spec.StorageCapacity, defaultStorageCapacity)), + corev1.ResourceStorage: Equal(utils.TypeDeref(s.etcd.Spec.StorageCapacity, defaultStorageCapacity)), }), }), "StorageClassName": Equal(s.etcd.Spec.StorageClass), @@ -149,7 +149,7 @@ func (s StatefulSetMatcher) matchPodSpec() gomegatypes.GomegaMatcher { "Containers": s.matchContainers(), "SecurityContext": s.matchEtcdPodSecurityContext(), "Volumes": s.matchPodVolumes(), - "PriorityClassName": Equal(utils.TypeDeref[string](s.etcd.Spec.PriorityClassName, "")), + "PriorityClassName": Equal(utils.TypeDeref(s.etcd.Spec.PriorityClassName, "")), }) } @@ -198,7 +198,7 @@ func (s StatefulSetMatcher) matchContainers() gomegatypes.GomegaMatcher { } func (s StatefulSetMatcher) matchEtcdContainer() gomegatypes.GomegaMatcher { - etcdContainerResources := utils.TypeDeref[corev1.ResourceRequirements](s.etcd.Spec.Etcd.Resources, defaultTestContainerResources) + etcdContainerResources := utils.TypeDeref(s.etcd.Spec.Etcd.Resources, defaultTestContainerResources) return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "Name": Equal("etcd"), "Image": Equal(s.etcdImage), @@ -266,9 +266,9 @@ func (s StatefulSetMatcher) matchEtcdContainerReadinessHandler() gomegatypes.Gom })), }) } - scheme := utils.IfConditionOr[corev1.URIScheme](s.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) - path := utils.IfConditionOr[string](s.etcd.Spec.Replicas > 1, "/readyz", "/healthz") - port := utils.IfConditionOr[int32](s.etcd.Spec.Replicas > 1, 9095, 8080) + scheme := utils.IfConditionOr(s.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) + path := utils.IfConditionOr(s.etcd.Spec.Replicas > 1, "/readyz", "/healthz") + port := utils.IfConditionOr(s.etcd.Spec.Replicas > 1, 9095, 8080) return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "HTTPGet": PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "Path": Equal(path), @@ -319,7 +319,7 @@ func (s StatefulSetMatcher) matchEtcdContainerCmdArgs() gomegatypes.GomegaMatche } func (s StatefulSetMatcher) matchEtcdDataVolMount() gomegatypes.GomegaMatcher { - volumeClaimTemplateName := utils.TypeDeref[string](s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) + volumeClaimTemplateName := utils.TypeDeref(s.etcd.Spec.VolumeClaimTemplate, s.etcd.Name) return matchVolMount(volumeClaimTemplateName, common.VolumeMountPathEtcdData) } @@ -383,7 +383,7 @@ func (s StatefulSetMatcher) matchEtcdContainerEnvVars() gomegatypes.GomegaMatche if s.useEtcdWrapper { return BeEmpty() } - scheme := utils.IfConditionOr[string](s.etcd.Spec.Backup.TLS != nil, "https", "http") + scheme := utils.IfConditionOr(s.etcd.Spec.Backup.TLS != nil, "https", "http") return ConsistOf( MatchFields(IgnoreExtras, Fields{ "Name": Equal("ENABLE_TLS"), diff --git a/internal/utils/store_test.go b/internal/utils/store_test.go index 61ee0e912..0010bec9c 100644 --- a/internal/utils/store_test.go +++ b/internal/utils/store_test.go @@ -85,7 +85,7 @@ func TestGetHostMountPathFromSecretRef(t *testing.T) { existingObjects = append(existingObjects, sec) } cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, client.ObjectKey{Name: existingSecretName, Namespace: testutils.TestNamespace}) - secretName := IfConditionOr[string](tc.secretExists, existingSecretName, nonExistingSecretName) + secretName := IfConditionOr(tc.secretExists, existingSecretName, nonExistingSecretName) storeSpec := createStoreSpec(tc.secretRefDefined, secretName, testutils.TestNamespace) actualHostPath, err := GetHostMountPathFromSecretRef(context.Background(), cl, logger, storeSpec, testutils.TestNamespace) if tc.expectedErr != nil { diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index 1e8b575a5..3416eb20a 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -131,7 +131,7 @@ func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegis if !slices.Equal(actualResourceNames, expectedResourceNames) { return fmt.Errorf("expected %s: %v, found %v instead", kind, expectedResourceNames, actualResourceNames) } - msg := utils.IfConditionOr[string](len(expectedResourceNames) == 0, + msg := utils.IfConditionOr(len(expectedResourceNames) == 0, fmt.Sprintf("%s: %v does not exist", kind, expectedResourceNames), fmt.Sprintf("%s: %v exists", kind, expectedResourceNames)) t.Log(msg) @@ -271,7 +271,7 @@ func assertETCDOperationAnnotation(t *testing.T, cl client.Client, etcdObjectKey return nil } g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) - msg := utils.IfConditionOr[string](expectedAnnotationToBePresent, "reconcile operation annotation present", "reconcile operation annotation removed") + msg := utils.IfConditionOr(expectedAnnotationToBePresent, "reconcile operation annotation present", "reconcile operation annotation removed") t.Log(msg) } @@ -307,12 +307,12 @@ func assertETCDFinalizer(t *testing.T, cl client.Client, etcdObjectKey client.Ob } finalizerPresent := slices.Contains(etcdInstance.ObjectMeta.Finalizers, common.FinalizerName) if expectedFinalizerPresent != finalizerPresent { - return fmt.Errorf("expected finalizer to be %s, found %v", utils.IfConditionOr[string](expectedFinalizerPresent, "present", "removed"), etcdInstance.ObjectMeta.Finalizers) + return fmt.Errorf("expected finalizer to be %s, found %v", utils.IfConditionOr(expectedFinalizerPresent, "present", "removed"), etcdInstance.ObjectMeta.Finalizers) } return nil } g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).Should(BeNil()) - msg := utils.IfConditionOr[string](expectedFinalizerPresent, "finalizer present", "finalizer removed") + msg := utils.IfConditionOr(expectedFinalizerPresent, "finalizer present", "finalizer removed") t.Log(msg) } @@ -388,7 +388,7 @@ func assertETCDStatusFieldsDerivedFromStatefulSet(ctx context.Context, t *testin if etcdInstance.Status.Replicas != *sts.Spec.Replicas { return fmt.Errorf("expected replicas to be %d, found %d", sts.Spec.Replicas, etcdInstance.Status.Replicas) } - if utils.TypeDeref[bool](etcdInstance.Status.Ready, false) != expectedReadyStatus { + if utils.TypeDeref(etcdInstance.Status.Ready, false) != expectedReadyStatus { return fmt.Errorf("expected ready to be %t, found %s", expectedReadyStatus, logPointerTypeToString[bool](etcdInstance.Status.Ready)) } return nil diff --git a/test/utils/etcd.go b/test/utils/etcd.go index a6f204f01..4d5757aa7 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -260,7 +260,7 @@ func (eb *EtcdBuilder) WithEtcdClientServiceLabels(labels map[string]string) *Et eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} } - eb.etcd.Spec.Etcd.ClientService.Labels = MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Labels, labels) + eb.etcd.Spec.Etcd.ClientService.Labels = MergeMaps(eb.etcd.Spec.Etcd.ClientService.Labels, labels) return eb } @@ -273,7 +273,7 @@ func (eb *EtcdBuilder) WithEtcdClientServiceAnnotations(annotations map[string]s eb.etcd.Spec.Etcd.ClientService = &druidv1alpha1.ClientService{} } - eb.etcd.Spec.Etcd.ClientService.Annotations = MergeMaps[string](eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) + eb.etcd.Spec.Etcd.ClientService.Annotations = MergeMaps(eb.etcd.Spec.Etcd.ClientService.Annotations, annotations) return eb } @@ -296,14 +296,14 @@ func (eb *EtcdBuilder) WithBackupPort(port *int32) *EtcdBuilder { // WithLabels merges the labels that already exists with the ones that are passed to this method. If an entry is // already present in the existing labels then it gets overwritten with the new value present in the passed in labels. func (eb *EtcdBuilder) WithLabels(labels map[string]string) *EtcdBuilder { - MergeMaps[string, string](eb.etcd.Labels, labels) + MergeMaps(eb.etcd.Labels, labels) return eb } // WithAnnotations merges the existing annotations if any with the annotations passed. // Any existing entry will be replaced by the one that is present in the annotations passed to this method. func (eb *EtcdBuilder) WithAnnotations(annotations map[string]string) *EtcdBuilder { - eb.etcd.Annotations = MergeMaps[string, string](eb.etcd.Annotations, annotations) + eb.etcd.Annotations = MergeMaps(eb.etcd.Annotations, annotations) return eb } From 31e3a01ab68b2a5c779709c09f8c2fc2ca22588a Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 9 May 2024 20:24:50 +0530 Subject: [PATCH 166/235] Address review comments by @ishan16696 --- .github/dependabot.yml | 4 ++++ Makefile | 10 +++++----- api/v1alpha1/types_constant.go | 4 ++-- api/v1alpha1/types_etcd.go | 8 ++++---- api/v1alpha1/types_etcd_test.go | 10 +++++----- .../10-crd-druid.gardener.cloud_etcds.yaml | 2 +- .../bases/10-crd-druid.gardener.cloud_etcds.yaml | 2 +- docs/concepts/controllers.md | 8 ++++---- docs/deployment/cli-flags.md | 2 +- ...ry-from-permanent-quorum-loss-in-etcd-cluster.md | 6 +++--- docs/proposals/02-snapshot-compaction.md | 13 +++++++------ docs/proposals/03-scaling-up-an-etcd-cluster.md | 11 ++++------- internal/common/constants.go | 2 +- internal/controller/compaction/config.go | 8 ++++++-- internal/controller/config.go | 12 ++++++------ .../controller/etcdcopybackupstask/reconciler.go | 12 ++++++------ internal/controller/manager.go | 10 +++++----- internal/operator/clientservice/clientservice.go | 2 +- internal/operator/configmap/configmap.go | 2 +- internal/operator/memberlease/memberlease.go | 2 +- internal/operator/peerservice/peerservice.go | 2 +- .../poddistruptionbudget/poddisruptionbudget.go | 2 +- internal/operator/role/role.go | 2 +- internal/operator/rolebinding/rolebinding.go | 2 +- internal/operator/serviceaccount/serviceaccount.go | 2 +- internal/operator/snapshotlease/snapshotlease.go | 4 ++-- internal/operator/statefulset/statefulset.go | 2 +- internal/operator/statefulset/stsmatcher.go | 2 +- main.go | 2 +- 29 files changed, 78 insertions(+), 72 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index da0acb85e..e5d42e3d5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +# +# SPDX-License-Identifier: Apache-2.0 + # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: diff --git a/Makefile b/Makefile index dc4f4714d..9993d9993 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ check-generate: .PHONY: generate generate: manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) @go generate "$(REPO_ROOT)/internal/..." - @"$(REPO_ROOT)/hack/update-codegen.sh" + @"$(HACK_DIR)/update-codegen.sh" # Build the docker image .PHONY: docker-build @@ -108,7 +108,7 @@ docker-push: .PHONY: test test: $(GINKGO) $(GOTESTFMT) # run ginkgo unit tests. These will be ported to golang native tests over a period of time. - @"$(REPO_ROOT)/hack/test.sh" ./api/... \ + @"$(HACK_DIR)/test.sh" ./api/... \ ./internal/controller/etcdcopybackupstask/... \ ./internal/controller/predicate/... \ ./internal/controller/secret/... \ @@ -116,7 +116,7 @@ test: $(GINKGO) $(GOTESTFMT) ./internal/mapper/... \ ./internal/metrics/... # run the golang native unit tests. - @TEST_COV="true" "$(REPO_ROOT)/hack/test-go.sh" ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... + @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... .PHONY: test-cov test-cov: $(GINKGO) $(SETUP_ENVTEST) @@ -132,8 +132,8 @@ test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) .PHONY: test-integration test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) - @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test.sh" ./test/integration/... - @SETUP_ENVTEST="true" "$(REPO_ROOT)/hack/test-go.sh" ./test/it/... + @SETUP_ENVTEST="true" "$(HACK_DIR)/test.sh" ./test/integration/... + @SETUP_ENVTEST="true" "$(HACK_DIR)/test-go.sh" ./test/it/... .PHONY: update-dependencies update-dependencies: diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/types_constant.go index ae09297da..62ce975ff 100644 --- a/api/v1alpha1/types_constant.go +++ b/api/v1alpha1/types_constant.go @@ -4,7 +4,7 @@ package v1alpha1 -// Common label keys to be placed on all druid managed resources +// Common label keys to be placed on all druid-managed resources const ( // LabelAppNameKey is a label which sets the name of the resource provisioned for an etcd cluster. LabelAppNameKey = "app.kubernetes.io/name" @@ -18,7 +18,7 @@ const ( LabelComponentKey = "app.kubernetes.io/component" ) -// Annotation keys that can be placed on an etcd custom resource. +// Annotation keys that can be placed on an Etcd custom resource. const ( // IgnoreReconciliationAnnotation is an annotation set by an operator in order to stop reconciliation. // Deprecated: Please use SuspendEtcdSpecReconcileAnnotation instead diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 1cd547105..15528c600 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -449,7 +449,7 @@ const ( LastOperationStateError LastOperationState = "Error" ) -// LastOperation holds the information on the last operation done on this resource. +// LastOperation holds the information on the last operation done on the Etcd resource. type LastOperation struct { // Type is the type of last operation. Type LastOperationType `json:"type"` @@ -463,7 +463,7 @@ type LastOperation struct { // as part of LastOperation aids in establishing this correlation. This further helps in also easily filtering // reconcile logs as all structured logs in a reconciliation run should have the `runID` referenced. RunID string `json:"runID"` - // LastUpdateTime is the time at which the operation was updated. + // LastUpdateTime is the time at which the operation was last updated. LastUpdateTime metav1.Time `json:"lastUpdateTime"` } @@ -590,9 +590,9 @@ func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { return nil } -// IsReconciliationSuspended returns true if the Etcd resource has the annotation set to suspend spec reconciliation, +// IsSpecReconciliationSuspended returns true if the Etcd resource has the annotation set to suspend spec reconciliation, // else returns false. -func (e *Etcd) IsReconciliationSuspended() bool { +func (e *Etcd) IsSpecReconciliationSuspended() bool { suspendReconcileAnnotKey := e.GetSuspendEtcdSpecReconcileAnnotationKey() return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) } diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 8f0c6ecbb..0bd9e6edf 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -177,13 +177,13 @@ var _ = Describe("Etcd", func() { }) }) - Context("IsReconciliationSuspended", func() { + Context("IsSpecReconciliationSuspended", func() { Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { It("should return true", func() { etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", } - Expect(etcd.IsReconciliationSuspended()).To(Equal(true)) + Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) }) Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { @@ -191,7 +191,7 @@ var _ = Describe("Etcd", func() { etcd.Annotations = map[string]string{ IgnoreReconciliationAnnotation: "true", } - Expect(etcd.IsReconciliationSuspended()).To(Equal(true)) + Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) }) Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { @@ -200,12 +200,12 @@ var _ = Describe("Etcd", func() { SuspendEtcdSpecReconcileAnnotation: "true", IgnoreReconciliationAnnotation: "true", } - Expect(etcd.IsReconciliationSuspended()).To(Equal(true)) + Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) }) Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { It("should return false", func() { - Expect(etcd.IsReconciliationSuspended()).To(Equal(false)) + Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(false)) }) }) }) diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index b09430040..ae9ffd86b 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -1872,7 +1872,7 @@ spec: type: string lastUpdateTime: description: LastUpdateTime is the time at which the operation - was updated. + was last updated. format: date-time type: string runID: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index b09430040..ae9ffd86b 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -1872,7 +1872,7 @@ spec: type: string lastUpdateTime: description: LastUpdateTime is the time at which the operation - was updated. + was last updated. format: date-time type: string runID: diff --git a/docs/concepts/controllers.md b/docs/concepts/controllers.md index 2dff0b070..60fece673 100644 --- a/docs/concepts/controllers.md +++ b/docs/concepts/controllers.md @@ -6,7 +6,7 @@ etcd-druid uses Kubebuilder to define the `Etcd` CR and its corresponding contro All controllers that are a part of etcd-druid reside in package `internal/controller`, as sub-packages. -etcd-druid currently consists of the following controllers, each having its own responsibility: +Etcd-druid currently consists of the following controllers, each having its own responsibility: - *etcd* : responsible for the reconciliation of the `Etcd` CR spec, which allows users to run etcd clusters within the specified Kubernetes cluster, and also responsible for periodically updating the `Etcd` CR status with the up-to-date state of the managed etcd cluster. - *compaction* : responsible for [snapshot compaction](/docs/proposals/02-snapshot-compaction.md). @@ -54,7 +54,7 @@ While building the controller, an event filter is set such that the behavior of The reason this filter is present is that any disruption in the `Etcd` resource due to reconciliation (due to changes in the `Etcd` spec, for example) while workloads are being run would cause unwanted downtimes to the etcd cluster. Hence, any user who wishes to avoid such disruptions, can choose to set the `--enable-etcd-spec-auto-reconcile` CLI flag to `false`. An example of this is Gardener's [gardenlet](https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md), which reconciles the `Etcd` resource only during a shoot cluster's [*maintenance window*](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_maintenance.md). -The controller adds a finalizer to the `Etcd` resource in order to ensure that it does not get deleted until all dependent resources managed by druid, aka managed components, are properly cleaned up. Only the *etcd controller* can delete a resource once it adds finalizers to it. This ensures that the proper deletion flow steps are followed while deleting the resource. During deletion flow, managed components are deleted in parallel. +The controller adds a finalizer to the `Etcd` resource in order to ensure that it does not get deleted until all dependent resources managed by etcd-druid, aka managed components, are properly cleaned up. Only the *etcd controller* can delete a resource once it adds finalizers to it. This ensures that the proper deletion flow steps are followed while deleting the resource. During deletion flow, managed components are deleted in parallel. ### `Etcd` Status Updates @@ -63,8 +63,8 @@ The `Etcd` resource status is updated periodically by `etcd controller`, the int Status fields of the `Etcd` resource such as `LastOperation`, `LastErrors` and `ObservedGeneration`, are updated to reflect the result of the recent reconciliation of the `Etcd` resource spec. - `LastOperation` holds information about the last operation performed on the etcd cluster, indicated by fields `Type`, `State`, `Description` and `LastUpdateTime`. Additionally, a field `RunID` indicates the unique ID assigned to the specific reconciliation run, to allow for better debugging of issues. -- `LastErrors` is a slice of errors encountered by the last reconciliation run. Each error consists of fields `Code` to indicate the custom druid error code for the error, a human-readable `Description`, and the `ObservedAt` time when the error was seen. -- `ObservedGeneration` indicates the latest `generation` of the `Etcd` resource that druid has "observed" and consequently reconciled. It helps identify whether a change in the `Etcd` resource spec was acted upon by druid or not. +- `LastErrors` is a slice of errors encountered by the last reconciliation run. Each error consists of fields `Code` to indicate the custom etcd-druid error code for the error, a human-readable `Description`, and the `ObservedAt` time when the error was seen. +- `ObservedGeneration` indicates the latest `generation` of the `Etcd` resource that etcd-druid has "observed" and consequently reconciled. It helps identify whether a change in the `Etcd` resource spec was acted upon by druid or not. Status fields of the `Etcd` resource which correspond to the `StatefulSet` like `CurrentReplicas`, `ReadyReplicas` and `Replicas` are updated to reflect those of the `StatefulSet` by the controller. diff --git a/docs/deployment/cli-flags.md b/docs/deployment/cli-flags.md index dec476f2f..526b57bff 100644 --- a/docs/deployment/cli-flags.md +++ b/docs/deployment/cli-flags.md @@ -23,7 +23,7 @@ Etcd-druid exposes the following CLI flags that allow for configuring its behavi | `etcd-member-notready-threshold` | `etcd-controller` | Threshold after which an etcd member is considered not ready if the status was unknown before. | `5m` | | `etcd-member-unknown-threshold` | `etcd-controller` | Threshold after which an etcd member is considered unknown. | `1m` | | `enable-backup-compaction` | `compaction-controller` | Enable automatic compaction of etcd backups. | `false` | -| `compaction-workers` | `compaction-controller` | Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. Setting this flag to 0 disables the controller. | `3` | +| `compaction-workers` | `compaction-controller` | Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. If compaction is enabled, the value for this flag must be greater than zero. | `3` | | `etcd-events-threshold` | `compaction-controller` | Total number of etcd events that can be allowed before a backup compaction job is triggered. | `1000000` | | `active-deadline-duration` | `compaction-controller` | Duration after which a running backup compaction job will be terminated. | `3h` | | `metrics-scrape-wait-duration` | `compaction-controller` | Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped. | `0s` | diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md index 347bbc755..9324126f2 100644 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -23,7 +23,7 @@ Target the control plane of affected shoot cluster via `kubectl`. Alternatively, 1. Add the following annotations to the `Etcd` resource `etcd-main`: 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile=` - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection=` + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/disable-resource-protection=` 2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: @@ -70,7 +70,7 @@ Target the control plane of affected shoot cluster via `kubectl`. Alternatively, 6. Edit the `etcd-main` cluster's configmap (ex: `etcd-bootstrap-4785b0`) as follows: - Find the `initial-cluster` field in the configmap. It will look like the following: + Find the `initial-cluster` field in the configmap. It should look similar to the following: ``` # Initial cluster initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380,etcd-main-1=https://etcd-main-1.etcd-main-peer.default.svc:2380,etcd-main-2=https://etcd-main-2.etcd-main-peer.default.svc:2380 @@ -99,7 +99,7 @@ Target the control plane of affected shoot cluster via `kubectl`. Alternatively, 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile-` - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/resource-protection-` + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/disable-resource-protection-` 10. Finally, add the following annotation to the `Etcd` resource `etcd-main`: diff --git a/docs/proposals/02-snapshot-compaction.md b/docs/proposals/02-snapshot-compaction.md index 4fb12eaa7..38494debd 100644 --- a/docs/proposals/02-snapshot-compaction.md +++ b/docs/proposals/02-snapshot-compaction.md @@ -32,12 +32,13 @@ To help with the problem mentioned earlier, our proposal is to introduce `compac ### How the solution works The newly introduced compact command does not disturb the running Etcd while compacting the backup snapshots. The command is designed to run potentially separately (from the main Etcd process/container/pod). Etcd Druid can be configured to run the newly introduced compact command as a separate job (scheduled periodically) based on total number of Etcd events accumulated after the most recent full snapshot. -### Druid flags: -Etcd druid introduced following flags to configure the compaction job: -- `--enable-backup-compaction` (default `false`): Set this flag to `true` to enable the automatic compaction of etcd backups when `etcd-events-threshold` is exceeded. -- `--compaction-workers` (default `3`): If this flag is set to zero, no compaction job will be running. If it's set to any value greater than zero, druid controller will have that many threads to kickstart the compaction job. -- `--etcd-events-threshold` (default `1000000`): Set this flag with the value which will signify the number of Etcd events allowed after the most recent full snapshot. Once the number of Etcd events crosses the value mentioned in this flag, compaction job will be kickstarted. -- `--active-deadline-duration` (default `3h`): This flag signifies the maximum duration till which a compaction job won't be garbage-collected. +### Etcd-druid flags: +Etcd-druid introduces the following flags to configure the compaction job: +- `--enable-backup-compaction` (default `false`): Set this flag to `true` to enable the automatic compaction of etcd backups when the threshold value denoted by CLI flag `--etcd-events-threshold` is exceeded. +- `--compaction-workers` (default `3`): Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. If compaction is enabled, the value for this flag must be greater than zero. +- `--etcd-events-threshold` (default `1000000`): Total number of etcd events that can be allowed before a backup compaction job is triggered. +- `--active-deadline-duration` (default `3h`): Duration after which a running backup compaction job will be terminated. +- `--metrics-scrape-wait-duration` (default `0s`): Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped. ### **Points to take care while saving the compacted snapshot:** As compacted snapshot and the existing periodic full snapshots are taken by different processes running in different pods but accessing same store to save the snapshots, some problems may arise: diff --git a/docs/proposals/03-scaling-up-an-etcd-cluster.md b/docs/proposals/03-scaling-up-an-etcd-cluster.md index 3f678c47a..998e15108 100644 --- a/docs/proposals/03-scaling-up-an-etcd-cluster.md +++ b/docs/proposals/03-scaling-up-an-etcd-cluster.md @@ -25,13 +25,10 @@ Now, it is detected whether peer URL was TLS enabled or not for single node etcd ## Action taken by etcd-druid to enable the peerURL TLS 1. Etcd-druid will update the `etcd-bootstrap` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. -2. In case the peer URL TLS has been changed to `enabled`: Etcd-druid will add tasks to the deployment flow. - - To ensure that the TLS enablement of peer URL is properly reflected in etcd, the existing etcd StatefulSet pods should be restarted twice. - - The first restart pushes a new configuration which contains Peer URL TLS configuration. Backup-restore will update the member peer url. This will result in the change of the peer url in the etcd's database, but it may not reflect in the already running etcd container. Ideally a restart of an etcd container would have been sufficient but currently k8s doesn't expose an API to force restart a single container within a pod. Therefore, we need to restart the StatefulSet pod(s) once again. When the pod(s) is restarted the second time it will now start etcd with the correct peer url which will be TLS enabled. - - To achieve 2 restarts following is done: - * An update is made to the spec mounting the peer URL TLS secrets. This will cause a rolling update of the existing pod. - * Once the update is successfully completed, then we delete StatefulSet pods, causing a restart by the StatefulSet controller. - +2. In case the peer URL TLS has been changed to `enabled`: Etcd-druid will add tasks to the deployment flow: + - Check if peer TLS has been enabled for existing StatefulSet pods, by checking the member leases for the annotation `member.etcd.gardener.cloud/tls-enabled`. + - If peer TLS enablement is pending for any of the members, then check and patch the StatefulSet with the peer TLS volume mounts, if not already patched. This will cause a rolling update of the existing StatefulSet pods, which allows etcd-backup-restore to update the member peer URL in the etcd cluster. + - Requeue this reconciliation flow until peer TLS has been enabled for all the existing etcd members. ## After PeerURL is TLS enabled After peer URL TLS enablement for single node etcd cluster, now etcd-druid adds a scale-up annotation: `gardener.cloud/scaled-to-multi-node` to the etcd statefulset and etcd-druid will patch the statefulsets `.spec.replicas` to `3`(for example). The statefulset controller will then bring up new pods(etcd with backup-restore as a sidecar). Now etcd's sidecar i.e backup-restore will check whether this member is already a part of a cluster or not and incase it is unable to check (may be due to some network issues) then backup-restore checks presence of this annotation: `gardener.cloud/scaled-to-multi-node` in etcd statefulset to detect scale-up. If it finds out it is the scale-up case then backup-restore adds new etcd member as a [learner](https://etcd.io/docs/v3.3/learning/learner/) first and then starts the etcd learner by providing the correct configuration. Once learner gets in sync with the etcd cluster leader, it will get promoted to a voting member. diff --git a/internal/common/constants.go b/internal/common/constants.go index de47b08c2..8ddad3325 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -167,6 +167,6 @@ const ( // VolumeMountPathNonGCSProviderBackupSecret is the path on a container where the non-GCS provider backup secret is mounted. VolumeMountPathNonGCSProviderBackupSecret = "/var/etcd-backup" - // VolumeMountPathEtcdData is the path on a container where the etcd data directory is hosted. + // VolumeMountPathEtcdData is the path on a container where the etcd data directory is mounted. VolumeMountPathEtcdData = "/var/etcd/data" ) diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 88a13a062..d0d5f40bc 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -54,7 +54,7 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.BoolVar(&cfg.EnableBackupCompaction, enableBackupCompactionFlagName, defaultEnableBackupCompaction, "Enable automatic compaction of etcd backups.") fs.IntVar(&cfg.Workers, workersFlagName, defaultCompactionWorkers, - "Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. Setting this flag to 0 disables the controller.") + "Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. If compaction is enabled, the value for this flag must be greater than zero.") fs.Int64Var(&cfg.EventsThreshold, eventsThresholdFlagName, defaultEventsThreshold, "Total number of etcd events that can be allowed before a backup compaction job is triggered.") fs.DurationVar(&cfg.ActiveDeadlineDuration, activeDeadlineDurationFlagName, defaultActiveDeadlineDuration, @@ -65,7 +65,11 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { // Validate validates the config. func (cfg *Config) Validate() error { - if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, 0, cfg.Workers); err != nil { + minWorkers := 0 + if cfg.EnableBackupCompaction { + minWorkers = 1 + } + if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, minWorkers, cfg.Workers); err != nil { return err } if err := utils.MustBeGreaterThan(eventsThresholdFlagName, 0, cfg.EventsThreshold); err != nil { diff --git a/internal/controller/config.go b/internal/controller/config.go index ed20be5dd..515a538d4 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -56,12 +56,12 @@ type ServerConfig struct { type HTTPSServer struct { // Server is the configuration for the bind address and the port. Server - // TLSServer contains information about the TLS configuration for an HTTPS server. - TLS TLSServer + // TLSConfig contains information about the TLS configuration for an HTTPS server. + TLSConfig TLSServerConfig } -// TLSServer contains information about the TLS configuration for an HTTPS server. -type TLSServer struct { +// TLSServerConfig contains information about the TLS configuration for an HTTPS server. +type TLSServerConfig struct { // ServerCertDir is the path to a directory containing the server's TLS certificate and key (the files must be // named tls.crt and tls.key respectively). ServerCertDir string @@ -128,7 +128,7 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { "The IP address on which to listen for the HTTPS webhook server.") flag.IntVar(&cfg.Server.Webhook.Server.Port, webhookServerPortFlagName, defaultWebhookServerPort, "The port on which to listen for the HTTPS webhook server.") - flag.StringVar(&cfg.Server.Webhook.TLS.ServerCertDir, webhookServerTLSServerCertDir, defaultWebhookServerTLSServerCert, + flag.StringVar(&cfg.Server.Webhook.TLSConfig.ServerCertDir, webhookServerTLSServerCertDir, defaultWebhookServerTLSServerCert, "The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively).") flag.BoolVar(&cfg.LeaderElection.Enabled, enableLeaderElectionFlagName, defaultEnableLeaderElection, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -198,7 +198,7 @@ func (cfg *ManagerConfig) Validate() error { if cfg.Server.Webhook.Port == 0 { return fmt.Errorf("webhook port cannot be 0") } - if cfg.Server.Webhook.TLS.ServerCertDir == "" { + if cfg.Server.Webhook.TLSConfig.ServerCertDir == "" { return fmt.Errorf("webhook server cert dir cannot be empty") } } diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index e2eb1b7ad..1c489b73c 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -504,16 +504,16 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum return } -func getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { - // "/var/etcd-backup" +func getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, volumeSuffix string) string { + // "/var/etcd-backup" tokens := strings.Split(strings.Trim(common.VolumeMountPathNonGCSProviderBackupSecret, "/"), "/") - return fmt.Sprintf("/%s/%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) + return fmt.Sprintf("/%s/%s%s%s", tokens[0], volumePrefix, tokens[1], volumeSuffix) } -func getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, suffix string) string { - // "/var/.gcp" +func getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, volumeSuffix string) string { + // "/var/.gcp" tokens := strings.Split(strings.TrimSuffix(common.VolumeMountPathGCSBackupSecret, "/"), ".") - return fmt.Sprintf("%s.%s%s%s", tokens[0], volumePrefix, tokens[1], suffix) + return fmt.Sprintf("%s.%s%s%s", tokens[0], volumePrefix, tokens[1], volumeSuffix) } // createEnvVarsFromStore generates a slice of environment variables for an EtcdCopyBackups job based on the given StoreSpec, diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 86fefc9b2..c11427f18 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -33,8 +33,8 @@ var ( defaultTimeout = time.Minute ) -// CreateManagerWithControllers creates a controller manager and adds all the controllers to the controller-manager using the passed in ManagerConfig. -func CreateManagerWithControllers(config *ManagerConfig) (ctrl.Manager, error) { +// CreateManagerWithControllersAndWebhooks creates a controller manager and adds all the controllers to the controller-manager using the passed in ManagerConfig. +func CreateManagerWithControllersAndWebhooks(config *ManagerConfig) (ctrl.Manager, error) { var ( err error mgr ctrl.Manager @@ -45,7 +45,7 @@ func CreateManagerWithControllers(config *ManagerConfig) (ctrl.Manager, error) { if mgr, err = createManager(config); err != nil { return nil, err } - if err = registerControllersWithManager(mgr, config); err != nil { + if err = registerControllersAndWebhooksWithManager(mgr, config); err != nil { return nil, err } @@ -83,7 +83,7 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { WebhookServer: webhook.NewServer(webhook.Options{ Host: config.Server.Webhook.BindAddress, Port: config.Server.Webhook.Port, - CertDir: config.Server.Webhook.TLS.ServerCertDir, + CertDir: config.Server.Webhook.TLSConfig.ServerCertDir, }), LeaderElection: config.LeaderElection.Enabled, LeaderElectionID: config.LeaderElection.ID, @@ -91,7 +91,7 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { }) } -func registerControllersWithManager(mgr ctrl.Manager, config *ManagerConfig) error { +func registerControllersAndWebhooksWithManager(mgr ctrl.Manager, config *ManagerConfig) error { var err error // Add etcd reconciler to the manager diff --git a/internal/operator/clientservice/clientservice.go b/internal/operator/clientservice/clientservice.go index 49376acf0..2763074e7 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/operator/clientservice/clientservice.go @@ -82,7 +82,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the client service for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of client service", "objectKey", objectKey) + ctx.Logger.Info("Triggering deletion of client service", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyClientService(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No Client Service found, Deletion is a No-Op", "objectKey", objectKey) diff --git a/internal/operator/configmap/configmap.go b/internal/operator/configmap/configmap.go index f0547866a..c03b140c0 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/operator/configmap/configmap.go @@ -91,7 +91,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the configmap for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of ConfigMap", "objectKey", objectKey) + ctx.Logger.Info("Triggering deletion of ConfigMap", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyConfigMap(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No ConfigMap found, Deletion is a No-Op", "objectKey", objectKey) diff --git a/internal/operator/memberlease/memberlease.go b/internal/operator/memberlease/memberlease.go index 50c4fb5e4..b623ea17d 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/operator/memberlease/memberlease.go @@ -103,7 +103,7 @@ func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1 // TriggerDelete deletes the member leases for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of member leases") + ctx.Logger.Info("Triggering deletion of member leases") if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(etcd.Namespace), diff --git a/internal/operator/peerservice/peerservice.go b/internal/operator/peerservice/peerservice.go index e8dfa8bfc..6f6984e59 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/operator/peerservice/peerservice.go @@ -82,7 +82,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the peer service for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of peer service") + ctx.Logger.Info("Triggering deletion of peer service") if err := r.client.Delete(ctx, emptyPeerService(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No Peer Service found, Deletion is a No-Op", "objectKey", objectKey) diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/operator/poddistruptionbudget/poddisruptionbudget.go index bea2ed3fc..403a18e33 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/operator/poddistruptionbudget/poddisruptionbudget.go @@ -81,7 +81,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the pod disruption budget for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of PDB") + ctx.Logger.Info("Triggering deletion of PDB") pdbObjectKey := getObjectKey(etcd) if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPodDisruptionBudget(pdbObjectKey))); err != nil { return druiderr.WrapError(err, diff --git a/internal/operator/role/role.go b/internal/operator/role/role.go index bb3bba6c9..1271ffd1c 100644 --- a/internal/operator/role/role.go +++ b/internal/operator/role/role.go @@ -82,7 +82,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the role for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) + ctx.Logger.Info("Triggering deletion of role", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyRole(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No Role found, Deletion is a No-Op", "objectKey", objectKey) diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/operator/rolebinding/rolebinding.go index 23abb37c2..947b7c704 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/operator/rolebinding/rolebinding.go @@ -82,7 +82,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the role binding for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of role", "objectKey", objectKey) + ctx.Logger.Info("Triggering deletion of role", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyRoleBinding(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No RoleBinding found, Deletion is a No-Op", "objectKey", objectKey) diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/operator/serviceaccount/serviceaccount.go index 4e173c919..c11aef500 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/operator/serviceaccount/serviceaccount.go @@ -83,7 +83,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the service account for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of service account") + ctx.Logger.Info("Triggering deletion of service account") objectKey := getObjectKey(etcd) if err := r.client.Delete(ctx, emptyServiceAccount(objectKey)); err != nil { if errors.IsNotFound(err) { diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 40da20154..6d233f2fd 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -80,7 +80,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the snapshot leases for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { - ctx.Logger.Info("Backup has been disabled. Triggering delete of snapshot leases") + ctx.Logger.Info("Backup has been disabled. Triggering deletion of snapshot leases") return r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { return druiderr.WrapError(err, ErrSyncSnapshotLease, @@ -106,7 +106,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the snapshot leases for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - ctx.Logger.Info("Triggering delete of snapshot leases") + ctx.Logger.Info("Triggering deletion of snapshot leases") if err := r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { return druiderr.WrapError(err, ErrDeleteSnapshotLease, diff --git a/internal/operator/statefulset/statefulset.go b/internal/operator/statefulset/statefulset.go index 7f1335dcb..cf01b7ee1 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/operator/statefulset/statefulset.go @@ -99,7 +99,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // TriggerDelete triggers the deletion of the statefulset for the given Etcd. func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd) - ctx.Logger.Info("Triggering delete of StatefulSet", "objectKey", objectKey) + ctx.Logger.Info("Triggering deletion of StatefulSet", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyStatefulSet(etcd)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No StatefulSet found, Deletion is a No-Op", "objectKey", objectKey.Name) diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/operator/statefulset/stsmatcher.go index ce700290e..a763a160c 100644 --- a/internal/operator/statefulset/stsmatcher.go +++ b/internal/operator/statefulset/stsmatcher.go @@ -268,7 +268,7 @@ func (s StatefulSetMatcher) matchEtcdContainerReadinessHandler() gomegatypes.Gom } scheme := utils.IfConditionOr(s.etcd.Spec.Backup.TLS == nil, corev1.URISchemeHTTP, corev1.URISchemeHTTPS) path := utils.IfConditionOr(s.etcd.Spec.Replicas > 1, "/readyz", "/healthz") - port := utils.IfConditionOr(s.etcd.Spec.Replicas > 1, 9095, 8080) + port := utils.IfConditionOr(s.etcd.Spec.Replicas > 1, int32(9095), int32(8080)) return MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "HTTPGet": PointTo(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "Path": Equal(path), diff --git a/main.go b/main.go index 575a669f3..28578697e 100644 --- a/main.go +++ b/main.go @@ -45,7 +45,7 @@ func main() { os.Exit(1) } - mgr, err := controller.CreateManagerWithControllers(&mgrConfig) + mgr, err := controller.CreateManagerWithControllersAndWebhooks(&mgrConfig) if err != nil { logger.Error(err, "failed to create druid controller manager") os.Exit(1) From c0895d233daee0e4aba48a2740cba1b90161adfd Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 13 May 2024 10:28:14 +0530 Subject: [PATCH 167/235] moved manager to its own package, addressed minor review comments --- api/v1alpha1/types_etcd.go | 6 ++-- api/v1alpha1/types_etcd_test.go | 26 ++++++++--------- internal/{controller => manager}/config.go | 12 ++++---- internal/{controller => manager}/manager.go | 32 ++++++++++----------- main.go | 6 ++-- 5 files changed, 40 insertions(+), 42 deletions(-) rename internal/{controller => manager}/config.go (96%) rename internal/{controller => manager}/manager.go (85%) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 15528c600..08e469ccd 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -247,7 +247,7 @@ type ClientService struct { // SharedConfig defines parameters shared and used by Etcd as well as backup-restore sidecar. type SharedConfig struct { - // AutoCompactionMode defines the auto-compaction-mode:'periodic' mode or 'revision' mode for etcd and embedded-Etcd of backup-restore sidecar. + // AutoCompactionMode defines the auto-compaction-mode:'periodic' mode or 'revision' mode for etcd and embedded-etcd of backup-restore sidecar. // +optional AutoCompactionMode *CompactionMode `json:"autoCompactionMode,omitempty"` // AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-etcd of backup-restore sidecar. @@ -597,8 +597,8 @@ func (e *Etcd) IsSpecReconciliationSuspended() bool { return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) } -// AreManagedResourcesProtected returns true if the Etcd resource has the disable-resource-protection annotation set, -// else returns false +// AreManagedResourcesProtected returns false if the Etcd resource has the disable-resource-protection annotation set, +// else returns true. func (e *Etcd) AreManagedResourcesProtected() bool { return !metav1.HasAnnotation(e.ObjectMeta, DisableResourceProtectionAnnotation) } diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 0bd9e6edf..5914383eb 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -144,7 +144,7 @@ var _ = Describe("Etcd", func() { Context("GetSuspendEtcdSpecReconcileAnnotationKey", func() { ignoreReconciliationAnnotation := IgnoreReconciliationAnnotation suspendEtcdSpecReconcileAnnotation := SuspendEtcdSpecReconcileAnnotation - Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { + Context("when etcd has only ignore-reconcile annotation set", func() { It("should return druid.gardener.cloud/ignore-reconciliation", func() { etcd.Annotations = map[string]string{ IgnoreReconciliationAnnotation: "true", @@ -152,8 +152,8 @@ var _ = Describe("Etcd", func() { Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&ignoreReconciliationAnnotation)) }) }) - Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { - It("should return druid.gardener.cloud/suspend-etcd-spec-reconcile", func() { + Context("when etcd has only suspend-etcd-spec-reconcile annotation set", func() { + It("should return suspend-etcd-spec-reconcile annotation", func() { etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", } @@ -161,7 +161,7 @@ var _ = Describe("Etcd", func() { }) }) Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { - It("should return druid.gardener.cloud/suspend-etcd-spec-reconcile", func() { + It("should return suspend-etcd-spec-reconcile annotation", func() { etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", IgnoreReconciliationAnnotation: "true", @@ -169,7 +169,7 @@ var _ = Describe("Etcd", func() { Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) }) }) - Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { + Context("when etcd does not have suspend-etcd-spec-reconcile or ignore-reconcile annotation set", func() { It("should return nil string pointer", func() { var nilString *string Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(nilString)) @@ -178,7 +178,7 @@ var _ = Describe("Etcd", func() { }) Context("IsSpecReconciliationSuspended", func() { - Context("when etcd has only annotation druid.gardener.cloud/suspend-etcd-spec-reconcile set", func() { + Context("when etcd has only suspend-etcd-spec-reconcile annotation set", func() { It("should return true", func() { etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", @@ -186,7 +186,7 @@ var _ = Describe("Etcd", func() { Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) }) - Context("when etcd has only annotation druid.gardener.cloud/ignore-reconciliation set", func() { + Context("when etcd has only ignore-reconcile annotation set", func() { It("should return true", func() { etcd.Annotations = map[string]string{ IgnoreReconciliationAnnotation: "true", @@ -194,7 +194,7 @@ var _ = Describe("Etcd", func() { Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) }) - Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { + Context("when etcd has both suspend-etcd-spec-reconcile and ignore-reconcile annotations set", func() { It("should return true", func() { etcd.Annotations = map[string]string{ SuspendEtcdSpecReconcileAnnotation: "true", @@ -203,7 +203,7 @@ var _ = Describe("Etcd", func() { Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) }) - Context("when etcd does not have annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { + Context("when etcd does not have suspend-etcd-spec-reconcile or annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { It("should return false", func() { Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(false)) }) @@ -261,10 +261,10 @@ var _ = Describe("Etcd", func() { func getEtcd(name, namespace string) *Etcd { var ( - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - metricLevel MetricsLevel = Basic + clientPort int32 = 2379 + serverPort int32 = 2380 + backupPort int32 = 8080 + metricLevel = Basic ) garbageCollectionPeriod := metav1.Duration{ diff --git a/internal/controller/config.go b/internal/manager/config.go similarity index 96% rename from internal/controller/config.go rename to internal/manager/config.go index 515a538d4..fe2663d02 100644 --- a/internal/controller/config.go +++ b/internal/manager/config.go @@ -87,8 +87,8 @@ type LeaderElectionConfig struct { ResourceLock string } -// ManagerConfig defines the configuration for the controller manager. -type ManagerConfig struct { +// Config defines the configuration for the controller manager. +type Config struct { // MetricsAddr is the address the metric endpoint binds to. // Deprecated: This field will be eventually removed. Please use Server.Metrics.BindAddress instead. MetricsAddr string @@ -112,7 +112,7 @@ type ManagerConfig struct { } // InitFromFlags initializes the controller manager config from the provided CLI flag set. -func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) error { cfg.Server = &ServerConfig{} cfg.Server.Metrics = &Server{} cfg.Server.Webhook = HTTPSServer{} @@ -162,7 +162,7 @@ func (cfg *ManagerConfig) InitFromFlags(fs *flag.FlagSet) error { } // initFeatureGates initializes feature gates from the provided CLI flag set. -func (cfg *ManagerConfig) initFeatureGates(fs *flag.FlagSet) error { +func (cfg *Config) initFeatureGates(fs *flag.FlagSet) error { featureGates := featuregate.NewFeatureGate() if err := featureGates.Add(features.GetDefaultFeatures()); err != nil { return fmt.Errorf("error adding features to the featuregate: %v", err) @@ -175,7 +175,7 @@ func (cfg *ManagerConfig) initFeatureGates(fs *flag.FlagSet) error { } // populateControllersFeatureGates adds relevant feature gates to every controller configuration -func (cfg *ManagerConfig) populateControllersFeatureGates() { +func (cfg *Config) populateControllersFeatureGates() { // Feature gates populated only for controllers that use feature gates // Add etcd controller feature gates @@ -189,7 +189,7 @@ func (cfg *ManagerConfig) populateControllersFeatureGates() { } // Validate validates the controller manager config. -func (cfg *ManagerConfig) Validate() error { +func (cfg *Config) Validate() error { if err := utils.ShouldBeOneOfAllowedValues("ResourceLock", getAllowedLeaderElectionResourceLocks(), cfg.LeaderElection.ResourceLock); err != nil { return err } diff --git a/internal/controller/manager.go b/internal/manager/manager.go similarity index 85% rename from internal/controller/manager.go rename to internal/manager/manager.go index c11427f18..6ef06a858 100644 --- a/internal/controller/manager.go +++ b/internal/manager/manager.go @@ -17,7 +17,6 @@ import ( "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" "github.com/gardener/etcd-druid/internal/webhook/sentinel" - coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" corev1 "k8s.io/api/core/v1" @@ -33,26 +32,26 @@ var ( defaultTimeout = time.Minute ) -// CreateManagerWithControllersAndWebhooks creates a controller manager and adds all the controllers to the controller-manager using the passed in ManagerConfig. -func CreateManagerWithControllersAndWebhooks(config *ManagerConfig) (ctrl.Manager, error) { +// InitializeManager creates a controller manager and adds all the controllers and webhooks to the controller-manager using the passed in Config. +func InitializeManager(config *Config) (ctrl.Manager, error) { var ( err error mgr ctrl.Manager ) - config.populateControllersFeatureGates() - if mgr, err = createManager(config); err != nil { return nil, err } - if err = registerControllersAndWebhooksWithManager(mgr, config); err != nil { + if err = registerControllers(mgr, config); err != nil { + return nil, err + } + if err = registerWebhooks(mgr, config); err != nil { return nil, err } - return mgr, nil } -func createManager(config *ManagerConfig) (ctrl.Manager, error) { +func createManager(config *Config) (ctrl.Manager, error) { // TODO: this can be removed once we have an improved informer, see https://github.com/gardener/etcd-druid/issues/215 // list of objects which should not be cached. uncachedObjects := []client.Object{ @@ -91,7 +90,7 @@ func createManager(config *ManagerConfig) (ctrl.Manager, error) { }) } -func registerControllersAndWebhooksWithManager(mgr ctrl.Manager, config *ManagerConfig) error { +func registerControllers(mgr ctrl.Manager, config *Config) error { var err error // Add etcd reconciler to the manager @@ -126,24 +125,23 @@ func registerControllersAndWebhooksWithManager(mgr ctrl.Manager, config *Manager // Add secret reconciler to the manager ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() - if err = secret.NewReconciler( + return secret.NewReconciler( mgr, config.SecretControllerConfig, - ).RegisterWithManager(ctx, mgr); err != nil { - return err - } + ).RegisterWithManager(ctx, mgr) +} +func registerWebhooks(mgr ctrl.Manager, config *Config) error { // Add sentinel webhook to the manager if config.SentinelWebhookConfig.Enabled { - var sentinelWebhook *sentinel.Handler - if sentinelWebhook, err = sentinel.NewHandler( + sentinelWebhook, err := sentinel.NewHandler( mgr, config.SentinelWebhookConfig, - ); err != nil { + ) + if err != nil { return err } return sentinelWebhook.RegisterWithManager(mgr) } - return nil } diff --git a/main.go b/main.go index 28578697e..9e9d96514 100644 --- a/main.go +++ b/main.go @@ -8,11 +8,11 @@ import ( "os" "runtime" + druidmgr "github.com/gardener/etcd-druid/internal/manager" "github.com/go-logr/logr" "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/gardener/etcd-druid/internal/controller" "github.com/gardener/etcd-druid/internal/version" flag "github.com/spf13/pflag" @@ -30,7 +30,7 @@ func main() { printVersionInfo() - mgrConfig := controller.ManagerConfig{} + mgrConfig := druidmgr.Config{} if err := mgrConfig.InitFromFlags(flag.CommandLine); err != nil { logger.Error(err, "failed to initialize from flags") os.Exit(1) @@ -45,7 +45,7 @@ func main() { os.Exit(1) } - mgr, err := controller.CreateManagerWithControllersAndWebhooks(&mgrConfig) + mgr, err := druidmgr.InitializeManager(&mgrConfig) if err != nil { logger.Error(err, "failed to create druid controller manager") os.Exit(1) From c1ea02b2acb462963f277d0e53a1534a68493f21 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 13 May 2024 11:59:41 +0530 Subject: [PATCH 168/235] removed new label from match labels as that will cause failure to do rolling update. --- charts/druid/templates/controller-deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index 1f29a21fa..b701c7706 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -10,11 +10,12 @@ spec: replicas: {{ .Values.replicas }} selector: matchLabels: - app.kubernetes.io/name: etcd-druid + gardener.cloud/role: etcd-druid template: metadata: labels: app.kubernetes.io/name: etcd-druid + gardener.cloud/role: etcd-druid spec: serviceAccountName: etcd-druid containers: From ae5877e5ef2fa457db0b4acb87819c2cdac229e9 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 13 May 2024 14:09:26 +0530 Subject: [PATCH 169/235] Minor changes to component names --- .../templates/10-crd-druid.gardener.cloud_etcds.yaml | 2 +- .../crd/bases/10-crd-druid.gardener.cloud_etcds.yaml | 2 +- internal/common/constants.go | 12 ++++++------ .../controller/etcdcopybackupstask/reconciler.go | 2 +- .../etcdcopybackupstask/reconciler_test.go | 4 ++-- main.go | 4 ++-- .../etcdcopybackupstask/reconciler_test.go | 4 ++-- test/utils/etcdcopybackupstask.go | 2 +- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index ae9ffd86b..a79711a35 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -1687,7 +1687,7 @@ spec: properties: autoCompactionMode: description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore + mode or 'revision' mode for etcd and embedded-etcd of backup-restore sidecar. enum: - periodic diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index ae9ffd86b..a79711a35 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -1687,7 +1687,7 @@ spec: properties: autoCompactionMode: description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore + mode or 'revision' mode for etcd and embedded-etcd of backup-restore sidecar. enum: - periodic diff --git a/internal/common/constants.go b/internal/common/constants.go index 8ddad3325..178d2dc9e 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -90,7 +90,7 @@ const ( // ComponentNameClientService is the component name for client service resource. ComponentNameClientService = "etcd-client-service" // ComponentNameConfigMap is the component name for config map resource. - ComponentNameConfigMap = "etcd-config" + ComponentNameConfigMap = "etcd-configmap" // ComponentNameMemberLease is the component name for member lease resource. ComponentNameMemberLease = "etcd-member-lease" // ComponentNameSnapshotLease is the component name for snapshot lease resource. @@ -100,17 +100,17 @@ const ( // ComponentNamePodDisruptionBudget is the component name for pod disruption budget resource. ComponentNamePodDisruptionBudget = "etcd-pdb" // ComponentNameRole is the component name for role resource. - ComponentNameRole = "etcd-druid-role" + ComponentNameRole = "etcd-role" // ComponentNameRoleBinding is the component name for role binding resource. - ComponentNameRoleBinding = "druid-role-binding" + ComponentNameRoleBinding = "etcd-role-binding" // ComponentNameServiceAccount is the component name for service account resource. - ComponentNameServiceAccount = "druid-service-account" + ComponentNameServiceAccount = "etcd-service-account" // ComponentNameStatefulSet is the component name for statefulset resource. ComponentNameStatefulSet = "etcd-sts" // ComponentNameCompactionJob is the component name for compaction job resource. ComponentNameCompactionJob = "etcd-compaction-job" - // ComponentNameEtcdCopyBackupsTask is the component name for copy-backup task resource. - ComponentNameEtcdCopyBackupsTask = "etcd-copy-backup-task" + // ComponentNameEtcdCopyBackupsJob is the component name for copy-backup task resource. + ComponentNameEtcdCopyBackupsJob = "etcd-copy-backups-job" ) // Constants for volume names diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 1c489b73c..ea1d2ea5f 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -387,7 +387,7 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et func getLabels(task *druidv1alpha1.EtcdCopyBackupsTask, includeNetworkPolicyLabels bool) map[string]string { labels := make(map[string]string) - labels[druidv1alpha1.LabelComponentKey] = common.ComponentNameEtcdCopyBackupsTask + labels[druidv1alpha1.LabelComponentKey] = common.ComponentNameEtcdCopyBackupsJob labels[druidv1alpha1.LabelPartOfKey] = task.Name labels[druidv1alpha1.LabelManagedByKey] = druidv1alpha1.LabelManagedByValue labels[druidv1alpha1.LabelAppNameKey] = task.GetJobName() diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index 0689d51eb..fedf64b19 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -733,7 +733,7 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsJob), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), @@ -753,7 +753,7 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Template": MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsJob), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), diff --git a/main.go b/main.go index 9e9d96514..5d0f5c04d 100644 --- a/main.go +++ b/main.go @@ -8,14 +8,14 @@ import ( "os" "runtime" - druidmgr "github.com/gardener/etcd-druid/internal/manager" "github.com/go-logr/logr" "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/log/zap" + druidmgr "github.com/gardener/etcd-druid/internal/manager" "github.com/gardener/etcd-druid/internal/version" - flag "github.com/spf13/pflag" + flag "github.com/spf13/pflag" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" // +kubebuilder:scaffold:imports diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index 1631db245..66218b20b 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -136,7 +136,7 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Name": Equal(task.Name + "-worker"), "Namespace": Equal(task.Namespace), "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsJob), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), @@ -156,7 +156,7 @@ func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.I "Template": MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Labels": MatchKeys(IgnoreExtras, Keys{ - druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsTask), + druidv1alpha1.LabelComponentKey: Equal(common.ComponentNameEtcdCopyBackupsJob), druidv1alpha1.LabelPartOfKey: Equal(task.Name), druidv1alpha1.LabelManagedByKey: Equal(druidv1alpha1.LabelManagedByValue), druidv1alpha1.LabelAppNameKey: Equal(task.GetJobName()), diff --git a/test/utils/etcdcopybackupstask.go b/test/utils/etcdcopybackupstask.go index 2bcb7deb7..5f9510505 100644 --- a/test/utils/etcdcopybackupstask.go +++ b/test/utils/etcdcopybackupstask.go @@ -110,7 +110,7 @@ func getLabels(taskName, jobName string, includeNetworkPolicyLabels bool) map[st labels := map[string]string{ druidv1alpha1.LabelPartOfKey: taskName, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, - druidv1alpha1.LabelComponentKey: common.ComponentNameEtcdCopyBackupsTask, + druidv1alpha1.LabelComponentKey: common.ComponentNameEtcdCopyBackupsJob, druidv1alpha1.LabelAppNameKey: jobName, } if includeNetworkPolicyLabels { From 694cb65b4301d6a22aaa80bef7331ca0cc84fe55 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 13 May 2024 17:00:48 +0530 Subject: [PATCH 170/235] Address review comments by @anveshreddy18 --- api/v1alpha1/types_etcd.go | 2 +- internal/webhook/sentinel/handler.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 08e469ccd..22ea655ca 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -594,7 +594,7 @@ func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { // else returns false. func (e *Etcd) IsSpecReconciliationSuspended() bool { suspendReconcileAnnotKey := e.GetSuspendEtcdSpecReconcileAnnotationKey() - return suspendReconcileAnnotKey != nil && metav1.HasAnnotation(e.ObjectMeta, *suspendReconcileAnnotKey) + return suspendReconcileAnnotKey != nil } // AreManagedResourcesProtected returns false if the Etcd resource has the disable-resource-protection annotation set, diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 18ae3fd4e..20b6134b8 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -83,32 +83,32 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R etcd := &druidv1alpha1.Etcd{} if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { if apierrors.IsNotFound(err) { - return admission.Allowed(fmt.Sprintf("corresponding etcd %s not found", etcdName)) + return admission.Allowed(fmt.Sprintf("corresponding Etcd %s not found", etcdName)) } return admission.Errored(http.StatusInternalServerError, err) } - // allow changes to resources if etcd has annotation druid.gardener.cloud/disable-resource-protection is set + // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set if !etcd.AreManagedResourcesProtected() { - return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) + return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } - // allow operations on resources if any etcd operation is currently being reconciled, but only by etcd-druid, - // and allow exempt service accounts to make changes to resources, but only if etcd is not currently being reconciled. + // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid, + // and allow exempt service accounts to make changes to resources, but only if the Etcd is not currently being reconciled. if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { - return admission.Allowed(fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", etcd.Name)) + return admission.Allowed(fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", etcd.Name)) } - return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", etcd.Name)) } else { for _, sa := range h.config.ExemptServiceAccounts { if req.UserInfo.Username == sa { - return admission.Allowed(fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) + return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) } } } - return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", etcd.Name)) } func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.GroupKind) (client.Object, error) { From 2102a45291ffad1f415d631521617fc5d189c4a7 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 14 May 2024 11:40:13 +0530 Subject: [PATCH 171/235] added support for ko, updated skaffold, added new makefile targets for skaffold dev, debug, changed handling of images.yaml --- Makefile | 12 + .../templates/controller-deployment.yaml | 4 +- hack/ci-e2e-kind.sh | 2 +- hack/e2e-test/run-e2e-test.sh | 23 +- hack/tools.mk | 11 +- internal/controller/compaction/reconciler.go | 4 +- internal/controller/etcd/reconcile_spec.go | 5 + internal/controller/etcd/reconciler.go | 3 +- internal/controller/etcd/register.go | 18 +- .../etcdcopybackupstask/reconciler.go | 4 +- internal/controller/utils/reconciler.go | 12 +- internal/images/embed.go | 21 ++ {charts => internal/images}/images.yaml | 0 skaffold.yaml | 266 +++++++++--------- 14 files changed, 215 insertions(+), 170 deletions(-) create mode 100644 internal/images/embed.go rename {charts => internal/images}/images.yaml (100%) diff --git a/Makefile b/Makefile index 9993d9993..16d7ac1af 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,18 @@ deploy-via-kustomize: manifests $(KUSTOMIZE) deploy: $(SKAFFOLD) $(HELM) $(SKAFFOLD) run -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) +.PHONY: deploy-dev +deploy-dev: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) dev -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) + +.PHONY: deploy-debug +deploy-debug: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) debug -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) + +.PHONY: undeploy +undeploy: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) delete -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) + # Generate manifests e.g. CRD, RBAC etc. .PHONY: manifests manifests: $(VGOPATH) $(CONTROLLER_GEN) diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index b701c7706..dc413d4ba 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -22,9 +22,7 @@ spec: - name: etcd-druid image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.imagePullPolicy }} - command: - - /etcd-druid - + args: {{- if .Values.featureGates }} {{- $featuregates := "" }} {{- range $feature, $value := $.Values.featureGates }} diff --git a/hack/ci-e2e-kind.sh b/hack/ci-e2e-kind.sh index c0fa3b431..b3096fbb3 100755 --- a/hack/ci-e2e-kind.sh +++ b/hack/ci-e2e-kind.sh @@ -10,7 +10,7 @@ set -o pipefail make kind-up trap '{ - kind export logs "${ARTIFACTS:-}/etcd-druid-e2e" --name etcd-druid-e2e || true + kind export logs "${ARTIFACTS:-/tmp}/etcd-druid-e2e" --name etcd-druid-e2e || true make kind-down }' EXIT diff --git a/hack/e2e-test/run-e2e-test.sh b/hack/e2e-test/run-e2e-test.sh index f35b086c7..c977fee0f 100755 --- a/hack/e2e-test/run-e2e-test.sh +++ b/hack/e2e-test/run-e2e-test.sh @@ -23,7 +23,7 @@ function skaffold_run_or_deploy { if [[ -n ${IMAGE_NAME:=""} ]] && [[ -n ${IMAGE_TEST_TAG:=""} ]]; then skaffold deploy --images ${IMAGE_NAME}:${IMAGE_TEST_TAG} $@ else - skaffold run $@ + skaffold run "$@" fi } @@ -54,54 +54,55 @@ function teardown_trap { } function cleanup { - if containsElement $STEPS "cleanup" && [[ $profile_setup != "" ]]; then + if containsElement "$STEPS" "cleanup" && [[ $profile_setup != "" ]]; then echo "-------------------" echo "Tearing down environment" echo "-------------------" create_namespace - skaffold_run_or_deploy -p ${profile_cleanup} -m druid-e2e -n $TEST_ID --status-check=false + skaffold_run_or_deploy -p "${profile_cleanup}" -m druid-e2e -n "$TEST_ID" --status-check=false fi cleanup_done="true" } function undeploy { - if containsElement $STEPS "undeploy"; then - skaffold delete -m etcd-druid -n $TEST_ID + if containsElement "$STEPS" "undeploy"; then + skaffold delete -m etcd-druid -n "$TEST_ID" delete_namespace fi undeploy_done="true" } function setup_e2e { + # shellcheck disable=SC2223 : ${profile_setup:=""} trap teardown_trap INT TERM - if containsElement $STEPS "setup" && [[ $profile_setup != "" ]]; then + if containsElement "$STEPS" "setup" && [[ $profile_setup != "" ]]; then echo "-------------------" echo "Setting up environment" echo "-------------------" create_namespace - skaffold_run_or_deploy -p ${profile_setup} -m druid-e2e -n $TEST_ID --status-check=false + skaffold_run_or_deploy -p ${profile_setup} -m druid-e2e -n "$TEST_ID" --status-check=false fi } function deploy { - if containsElement $STEPS "deploy"; then + if containsElement "$STEPS" "deploy"; then if [[ ${deployed:="false"} != "true" ]] || true; then echo "-------------------" echo "Deploying Druid" echo "-------------------" export DRUID_E2E_TEST=true - skaffold_run_or_deploy -m etcd-druid -n $TEST_ID + skaffold_run_or_deploy -m etcd-druid -n "$TEST_ID" deployed="true" fi fi } function test_e2e { - if containsElement $STEPS "test"; then + if containsElement "$STEPS" "test"; then echo "-------------------" echo "Running e2e tests" echo "-------------------" @@ -149,6 +150,7 @@ EOM } function setup_azure_e2e { + # shellcheck disable=SC2235 ( [[ -z ${STORAGE_ACCOUNT:-""} ]] || [[ -z ${STORAGE_KEY:=""} ]] || [[ -z ${TEST_ID:=""} ]] ) && usage_azure profile_setup="azure-setup" @@ -173,6 +175,7 @@ EOM function setup_gcp_e2e { ( [[ -z ${GCP_SERVICEACCOUNT_JSON_PATH:-""} ]] || [[ -z ${GCP_PROJECT_ID:=""} ]] || [[ -z ${TEST_ID:=""} ]] ) && usage_gcp + # shellcheck disable=SC2031 export GOOGLE_APPLICATION_CREDENTIALS=$GCP_SERVICEACCOUNT_JSON_PATH profile_setup="gcp-setup" profile_cleanup="gcp-cleanup" diff --git a/hack/tools.mk b/hack/tools.mk index f48038068..a855bbd80 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -3,14 +3,17 @@ # SPDX-License-Identifier: Apache-2.0 TOOLS_BIN_DIR := $(TOOLS_DIR)/bin -SKAFFOLD := $(TOOLS_BIN_DIR)/skaffold KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize GOTESTFMT := $(TOOLS_BIN_DIR)/gotestfmt +SKAFFOLD := $(TOOLS_BIN_DIR)/skaffold + +SYSTEM_NAME := $(shell uname -s | tr '[:upper:]' '[:lower:]') +SYSTEM_ARCH := $(shell uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') # default tool versions -SKAFFOLD_VERSION ?= v1.38.0 -KUSTOMIZE_VERSION ?= v4.5.7 +KUSTOMIZE_VERSION := v4.5.7 GOTESTFMT_VERSION ?= v2.5.0 +SKAFFOLD_VERSION := v2.11.1 export TOOLS_BIN_DIR := $(TOOLS_BIN_DIR) export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) @@ -23,4 +26,4 @@ $(KUSTOMIZE): @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v4@${KUSTOMIZE_VERSION} $(GOTESTFMT): - GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@$(GOTESTFMT_VERSION) + GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@$(GOTESTFMT_VERSION) \ No newline at end of file diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index dda0e8ece..03c13a2c1 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -12,8 +12,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" + "github.com/gardener/etcd-druid/internal/images" druidmetrics "github.com/gardener/etcd-druid/internal/metrics" "github.com/gardener/etcd-druid/internal/utils" @@ -50,7 +50,7 @@ type Reconciler struct { // NewReconciler creates a new reconciler for Compaction func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := ctrlutils.CreateImageVector() + imageVector, err := images.CreateImageVector() if err != nil { return nil, err } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index c6f014af5..db3ba0faa 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -133,6 +133,11 @@ func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { return true } + // If the observed generation is nil then it indicates that the Etcd is new and therefore spec reconciliation should be allowed. + if etcd.Status.ObservedGeneration == nil { + return true + } + // Default case: Do not reconcile. return false } diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index c947edf0d..2e5d7ebec 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -9,6 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + "github.com/gardener/etcd-druid/internal/images" "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/internal/operator/clientservice" "github.com/gardener/etcd-druid/internal/operator/component" @@ -45,7 +46,7 @@ type Reconciler struct { // NewReconciler creates a new reconciler for Etcd. func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := ctrlutils.CreateImageVector() + imageVector, err := images.CreateImageVector() if err != nil { return nil, err } diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 2913639a8..e0c95dd75 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -83,7 +83,9 @@ func (r *Reconciler) hasReconcileAnnotation() predicate.Predicate { UpdateFunc: func(updateEvent event.UpdateEvent) bool { return updateEvent.ObjectNew.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile }, - CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + CreateFunc: func(createEvent event.CreateEvent) bool { + return true + }, DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, } @@ -94,7 +96,9 @@ func (r *Reconciler) autoReconcileEnabled() predicate.Predicate { UpdateFunc: func(updateEvent event.UpdateEvent) bool { return r.config.EnableEtcdSpecAutoReconcile || r.config.IgnoreOperationAnnotation }, - CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + CreateFunc: func(createEvent event.CreateEvent) bool { + return true + }, DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, } @@ -105,7 +109,9 @@ func specUpdated() predicate.Predicate { UpdateFunc: func(updateEvent event.UpdateEvent) bool { return hasSpecChanged(updateEvent) }, - CreateFunc: func(createEvent event.CreateEvent) bool { return true }, + CreateFunc: func(createEvent event.CreateEvent) bool { + return true + }, DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, } @@ -117,8 +123,10 @@ func noSpecAndStatusUpdated() predicate.Predicate { return !hasSpecChanged(updateEvent) && !hasStatusChanged(updateEvent) }, GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, - CreateFunc: func(createEvent event.CreateEvent) bool { return true }, - DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, + CreateFunc: func(createEvent event.CreateEvent) bool { + return true + }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, } } diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index ea1d2ea5f..91f3a2ebd 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -12,8 +12,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" + "github.com/gardener/etcd-druid/internal/images" "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -52,7 +52,7 @@ type Reconciler struct { // NewReconciler creates a new reconciler for EtcdCopyBackupsTask. func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { - imageVector, err := ctrlutils.CreateImageVector() + imageVector, err := images.CreateImageVector() if err != nil { return nil, err } diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 0c1af71fa..a007d0446 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -6,27 +6,17 @@ package utils import ( "context" + _ "embed" "errors" "fmt" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/gardener/pkg/utils/imagevector" apierrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) -// CreateImageVector creates an image vector from the default images.yaml file or the images-wrapper.yaml file. -func CreateImageVector() (imagevector.ImageVector, error) { - imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(common.DefaultImageVectorFilePath) - if err != nil { - return nil, err - } - return imageVector, nil -} - // GetLatestEtcd returns the latest version of the Etcd object. func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ReconcileStepResult { if err := client.Get(ctx, objectKey, etcd); err != nil { diff --git a/internal/images/embed.go b/internal/images/embed.go new file mode 100644 index 000000000..fa5d8cc58 --- /dev/null +++ b/internal/images/embed.go @@ -0,0 +1,21 @@ +package images + +import ( + _ "embed" + + "github.com/gardener/gardener/pkg/utils/imagevector" +) + +var ( + //go:embed images.yaml + imagesYAML string +) + +// CreateImageVector creates an image vector from the default images.yaml file or the images-wrapper.yaml file. +func CreateImageVector() (imagevector.ImageVector, error) { + imgVec, err := imagevector.Read([]byte(imagesYAML)) + if err != nil { + return nil, err + } + return imagevector.WithEnvOverride(imgVec) +} diff --git a/charts/images.yaml b/internal/images/images.yaml similarity index 100% rename from charts/images.yaml rename to internal/images/images.yaml diff --git a/skaffold.yaml b/skaffold.yaml index a4ad90abb..9b942cab9 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1,154 +1,158 @@ --- -apiVersion: skaffold/v2beta25 +apiVersion: skaffold/v4beta10 kind: Config metadata: name: etcd-druid build: - local: - useBuildkit: true + local: { } artifacts: - - image: europe-docker.pkg.dev/gardener-project/public/gardener/etcd-druid - docker: - dockerfile: Dockerfile - target: druid + - image: europe-docker.pkg.dev/gardener-project/public/gardener/etcd-druid + ko: + fromImage: gcr.io/distroless/static-debian11:nonroot + dependencies: + paths: + - api + - internal + - go.mod + - VERSION + flags: + - -v deploy: helm: releases: - - name: etcd-druid - chartPath: charts/druid - namespace: default - artifactOverrides: - image: europe-docker.pkg.dev/gardener-project/public/gardener/etcd-druid - imageStrategy: - helm: {} - skipBuildDependencies: true - setValues: - controllers: - compaction: - etcdEventsThreshold: 15 - metricsScrapeWaitDuration: 30s - webhooks: - sentinel: - enabled: true + - name: etcd-druid + chartPath: charts/druid + namespace: default + skipBuildDependencies: true + setValues: + controllers: + compaction: + etcdEventsThreshold: 15 + metricsScrapeWaitDuration: 30s + webhooks: + sentinel: + enabled: true profiles: -- name: e2e-test - activation: - - env: "DRUID_E2E_TEST=true" - patches: - - op: add - path: /deploy/helm/releases/0/setValues/controllers - value: - etcd: - etcdStatusSyncPeriod: 5s -- name: do-not-use-feature-gates - activation: - - env: "USE_ETCD_DRUID_FEATURE_GATES=false" - patches: - - op: add - path: /deploy/helm/releases/0/setValues/featureGates - value: - UseEtcdWrapper: false + - name: e2e-test + activation: + - env: "DRUID_E2E_TEST=true" + patches: + - op: add + path: /deploy/helm/releases/0/setValues/controllers + value: + etcd: + etcdStatusSyncPeriod: 5s + - name: do-not-use-feature-gates + activation: + - env: "USE_ETCD_DRUID_FEATURE_GATES=false" + patches: + - op: add + path: /deploy/helm/releases/0/setValues/featureGates + value: + UseEtcdWrapper: false --- -apiVersion: skaffold/v2beta25 +apiVersion: skaffold/v4beta10 kind: Config metadata: name: druid-e2e deploy: - kustomize: + kubectl: hooks: before: - - host: - command: - - sh - - -c - - | - echo "Deleting previous job" - namespace_flag=${SKAFFOLD_NAMESPACES:-""} - if [ -n "$namespace_flag" ]; then - namespace_flag="-n ${namespace_flag}" - fi - kubectl delete job -l role=infra-job --ignore-not-found=true $namespace_flag + - host: + command: + - sh + - -c + - | + echo "Deleting previous job" + namespace_flag=${SKAFFOLD_NAMESPACES:-""} + if [ -n "$namespace_flag" ]; then + namespace_flag="-n ${namespace_flag}" + fi + kubectl delete job -l role=infra-job --ignore-not-found=true $namespace_flag after: - - host: - command: - - sh - - -c - - | - echo "Wait until job is ready" - namespace_flag=${SKAFFOLD_NAMESPACES:-""} - if [ -n "$namespace_flag" ]; then - namespace_flag="-n ${namespace_flag}" - fi - kubectl wait --for=condition=Complete job -l role=infra-job --timeout=5m $namespace_flag - os: [ darwin, linux ] + - host: + command: + - sh + - -c + - | + echo "Wait until job is ready" + namespace_flag=${SKAFFOLD_NAMESPACES:-""} + if [ -n "$namespace_flag" ]; then + namespace_flag="-n ${namespace_flag}" + fi + kubectl wait --for=condition=Complete job -l role=infra-job --timeout=5m $namespace_flag + os: [ darwin, linux ] +manifests: + kustomize: { } profiles: -# Profile to create the required S3 backup bucket for an e2e test. -- name: aws-setup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/aws/setup] -# Profile to delete the S3 backup bucket from an e2e test. -- name: aws-cleanup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/aws/cleanup] -# Profile to create the required Azure storage container for an e2e test. -- name: azure-setup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/azure/setup] -# Profile to delete the Azure storage container from an e2e test. -- name: azure-cleanup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/azure/cleanup] -# Profile to create the required GCP backup bucket for an e2e test. -- name: gcp-setup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/gcp/setup] - - op: add - path: /deploy/kustomize/hooks/before/- - value: { - "host": { - "command": [sh, -c, ' + # Profile to create the required S3 backup bucket for an e2e test. + - name: aws-setup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/aws/setup ] + # Profile to delete the S3 backup bucket from an e2e test. + - name: aws-cleanup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/aws/cleanup ] + # Profile to create the required Azure storage container for an e2e test. + - name: azure-setup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/azure/setup ] + # Profile to delete the Azure storage container from an e2e test. + - name: azure-cleanup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/azure/cleanup ] + # Profile to create the required GCP backup bucket for an e2e test. + - name: gcp-setup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/gcp/setup ] + - op: add + path: /manifests/kustomize/hooks/before/- + value: { + "host": { + "command": [ sh, -c, ' echo "Copying GCP service account json" && touch "hack/e2e-test/infrastructure/overlays/gcp/common/assets/serviceaccount.json" && - cp "$GCP_SERVICEACCOUNT_JSON_PATH" "hack/e2e-test/infrastructure/overlays/gcp/common/assets/serviceaccount.json"'], - "os": [darwin, linux] - } - } -# Profile to delete the GCP backup bucket from an e2e test. -- name: gcp-cleanup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/gcp/cleanup] - - op: add - path: /deploy/kustomize/hooks/before/- - value: { - "host": { - "command": [sh, -c, ' + cp "$GCP_SERVICEACCOUNT_JSON_PATH" "hack/e2e-test/infrastructure/overlays/gcp/common/assets/serviceaccount.json"' ], + "os": [ darwin, linux ] + } + } + # Profile to delete the GCP backup bucket from an e2e test. + - name: gcp-cleanup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/gcp/cleanup ] + - op: add + path: /manifests/kustomize/hooks/before/- + value: { + "host": { + "command": [ sh, -c, ' echo "Copying GCP service account json" && touch "hack/e2e-test/infrastructure/overlays/gcp/common/assets/serviceaccount.json" && - cp "$GCP_SERVICEACCOUNT_JSON_PATH" "hack/e2e-test/infrastructure/overlays/gcp/common/assets/serviceaccount.json"'], - "os": [darwin, linux] - } - } -# Profile to create the required Local storage container for an e2e test. -- name: local-setup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/local/setup] -# Profile to delete the Local storage container from an e2e test. -- name: local-cleanup - patches: - - op: add - path: /deploy/kustomize/paths - value: [hack/e2e-test/infrastructure/overlays/local/cleanup] + cp "$GCP_SERVICEACCOUNT_JSON_PATH" "hack/e2e-test/infrastructure/overlays/gcp/common/assets/serviceaccount.json"' ], + "os": [ darwin, linux ] + } + } + # Profile to create the required Local storage container for an e2e test. + - name: local-setup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/local/setup ] + # Profile to delete the Local storage container from an e2e test. + - name: local-cleanup + patches: + - op: add + path: /manifests/kustomize/paths + value: [ hack/e2e-test/infrastructure/overlays/local/cleanup ] From 3415a9d20370bcfd129a90c9744fc99676b315c9 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 14 May 2024 12:28:28 +0530 Subject: [PATCH 172/235] fixed unit test for sentil webhook --- internal/webhook/sentinel/handler.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 20b6134b8..511932a27 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -83,32 +83,32 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R etcd := &druidv1alpha1.Etcd{} if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { if apierrors.IsNotFound(err) { - return admission.Allowed(fmt.Sprintf("corresponding Etcd %s not found", etcdName)) + return admission.Allowed(fmt.Sprintf("corresponding etcd %s not found", etcdName)) } return admission.Errored(http.StatusInternalServerError, err) } // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set if !etcd.AreManagedResourcesProtected() { - return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) + return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid, // and allow exempt service accounts to make changes to resources, but only if the Etcd is not currently being reconciled. if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { - return admission.Allowed(fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", etcd.Name)) + return admission.Allowed(fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", etcd.Name)) } - return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", etcd.Name)) } else { for _, sa := range h.config.ExemptServiceAccounts { if req.UserInfo.Username == sa { - return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) + return admission.Allowed(fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) } } } - return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) } func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.GroupKind) (client.Object, error) { From 46b2c56df3bc5ee7281bb4903fdcab8c1ae702bd Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 14 May 2024 14:03:47 +0530 Subject: [PATCH 173/235] aligned messages in sentil to use Etcd instead of etcd --- internal/webhook/sentinel/handler.go | 12 +++---- internal/webhook/sentinel/handler_test.go | 40 +++++++++++------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 511932a27..20b6134b8 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -83,32 +83,32 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R etcd := &druidv1alpha1.Etcd{} if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { if apierrors.IsNotFound(err) { - return admission.Allowed(fmt.Sprintf("corresponding etcd %s not found", etcdName)) + return admission.Allowed(fmt.Sprintf("corresponding Etcd %s not found", etcdName)) } return admission.Errored(http.StatusInternalServerError, err) } // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set if !etcd.AreManagedResourcesProtected() { - return admission.Allowed(fmt.Sprintf("changes allowed, since etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) + return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid, // and allow exempt service accounts to make changes to resources, but only if the Etcd is not currently being reconciled. if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { - return admission.Allowed(fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", etcd.Name)) + return admission.Allowed(fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", etcd.Name)) } - return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", etcd.Name)) } else { for _, sa := range h.config.ExemptServiceAccounts { if req.UserInfo.Username == sa { - return admission.Allowed(fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) + return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) } } } - return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", etcd.Name)) + return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", etcd.Name)) } func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.GroupKind) (client.Object, error) { diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index 1084489a3..c22f72eb4 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -206,17 +206,17 @@ func TestHandleUpdate(t *testing.T) { objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, expectedAllowed: true, - expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { - name: "operator makes a request when etcd is being reconciled by druid", + name: "operator makes a request when Etcd is being reconciled by druid", userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: false, - expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, { @@ -226,27 +226,27 @@ func TestHandleUpdate(t *testing.T) { etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: true, - expectedMessage: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), + expectedMessage: fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", testEtcdName), expectedCode: http.StatusOK, }, { - name: "etcd is not currently being reconciled by druid, and request is from exempt service account", + name: "Etcd is not currently being reconciled by druid, and request is from exempt service account", userName: exemptServiceAccounts[0], objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, - expectedMessage: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { - name: "etcd is not currently being reconciled by druid, and request is from non-exempt service account", + name: "Etcd is not currently being reconciled by druid, and request is from non-exempt service account", userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: false, - expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, } @@ -385,14 +385,14 @@ func TestEtcdGetFailures(t *testing.T) { expectedCode int32 }{ { - name: "should allow when etcd is not found", + name: "should allow when Etcd is not found", etcdGetErr: apiNotFoundErr, expectedAllowed: true, - expectedReason: fmt.Sprintf("corresponding etcd %s not found", testEtcdName), + expectedReason: fmt.Sprintf("corresponding Etcd %s not found", testEtcdName), expectedCode: http.StatusOK, }, { - name: "error in getting etcd", + name: "error in getting Etcd", etcdGetErr: apiInternalErr, expectedAllowed: false, expectedMessage: errInternal.Error(), @@ -465,48 +465,48 @@ func TestHandleDelete(t *testing.T) { objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, expectedAllowed: true, - expectedMessage: fmt.Sprintf("changes allowed, since etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { - name: "etcd is currently being reconciled by druid", + name: "Etcd is currently being reconciled by druid", userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: false, expectedReason: "Forbidden", - expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", testEtcdName), expectedCode: http.StatusForbidden, }, { - name: "etcd is currently being reconciled by druid, but request is from druid", + name: "Etcd is currently being reconciled by druid, but request is from druid", userName: reconcilerServiceAccount, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: true, - expectedMessage: fmt.Sprintf("ongoing reconciliation of etcd %s by etcd-druid requires changes to resources", testEtcdName), + expectedMessage: fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", testEtcdName), expectedCode: http.StatusOK, }, { - name: "etcd is not currently being reconciled by druid, and request is from exempt service account", + name: "Etcd is not currently being reconciled by druid, and request is from exempt service account", userName: exemptServiceAccounts[0], objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, - expectedMessage: fmt.Sprintf("operations on etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { - name: "etcd is not currently being reconciled by druid, and request is from non-exempt service account", + name: "Etcd is not currently being reconciled by druid, and request is from non-exempt service account", userName: testUserName, objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: false, - expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of etcd %s by etcd-druid", testEtcdName), + expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", testEtcdName), expectedReason: "Forbidden", expectedCode: http.StatusForbidden, }, From 1eb40ade43376ce3632d015315ad7d66c2612d1f Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 14 May 2024 14:04:40 +0530 Subject: [PATCH 174/235] corrected the creation of image vector for IT tests --- test/integration/controllers/assets/assets.go | 5 ++--- test/it/controller/assets/assets.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/integration/controllers/assets/assets.go b/test/integration/controllers/assets/assets.go index 0105535dc..2754c9194 100644 --- a/test/integration/controllers/assets/assets.go +++ b/test/integration/controllers/assets/assets.go @@ -7,7 +7,7 @@ package assets import ( "path/filepath" - "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/images" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/gomega" @@ -25,8 +25,7 @@ func GetEtcdCopyBackupsTaskCrdPath() string { // CreateImageVector creates an image vector. func CreateImageVector() imagevector.ImageVector { - imageVectorPath := filepath.Join("..", "..", "..", "..", common.DefaultImageVectorFilePath) - imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(imageVectorPath) + imageVector, err := images.CreateImageVector() Expect(err).To(BeNil()) return imageVector } diff --git a/test/it/controller/assets/assets.go b/test/it/controller/assets/assets.go index 34f5179d2..d3493de0d 100644 --- a/test/it/controller/assets/assets.go +++ b/test/it/controller/assets/assets.go @@ -7,7 +7,7 @@ package assets import ( "path/filepath" - "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/images" "github.com/gardener/gardener/pkg/utils/imagevector" . "github.com/onsi/gomega" @@ -35,8 +35,7 @@ func GetEtcdChartPath() string { // CreateImageVector creates an image vector. func CreateImageVector(g *WithT) imagevector.ImageVector { - imageVectorPath := filepath.Join("..", "..", "..", "..", common.DefaultImageVectorFilePath) - imageVector, err := imagevector.ReadGlobalImageVectorWithEnvOverride(imageVectorPath) + imageVector, err := images.CreateImageVector() g.Expect(err).To(BeNil()) return imageVector } From dda7846c8c2f1eafbadb3985c780ca8b7b57e419 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 14:21:33 +0530 Subject: [PATCH 175/235] Fix integration tests, leaked envtest instances --- test/it/controller/etcd/assertions.go | 15 ++++ test/it/controller/etcd/reconciler_test.go | 85 +++++++++++++++++++--- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index 3416eb20a..ab652a524 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -196,6 +196,21 @@ func assertStatefulSetCreated(ctx component.OperatorContext, t *testing.T, opReg assertResourceCreation(ctx, t, opRegistry, operator.StatefulSetKind, etcd, expectedSTSNames, timeout, pollInterval) } +func assertStatefulSetGeneration(ctx context.Context, t *testing.T, cl client.Client, stsObjectKey client.ObjectKey, expectedGeneration int64, timeout, pollInterval time.Duration) { + g := NewWithT(t) + checkFn := func() error { + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, stsObjectKey, sts); err != nil { + return err + } + if sts.Generation != expectedGeneration { + return fmt.Errorf("expected sts generation to be %d, got %d", expectedGeneration, sts.Generation) + } + return nil + } + g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).WithContext(ctx).Should(BeNil()) +} + func assertETCDObservedGeneration(t *testing.T, cl client.Client, etcdObjectKey client.ObjectKey, expectedObservedGeneration *int64, timeout, pollInterval time.Duration) { g := NewWithT(t) checkFn := func() error { diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 6089b6e20..512084415 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -53,8 +53,11 @@ func TestMain(m *testing.M) { _, _ = fmt.Fprintf(os.Stderr, "failed to create integration test environment: %v\n", err) os.Exit(1) } - defer itTestEnvCloser() - os.Exit(m.Run()) + + // os.Exit() does not respect defer statements + exitCode := m.Run() + itTestEnvCloser() + os.Exit(exitCode) } // ------------------------------ reconcile spec tests ------------------------------ @@ -67,7 +70,8 @@ func TestEtcdReconcileSpecWithNoAutoReconcile(t *testing.T) { {"should create all managed resources when etcd resource is created", testAllManagedResourcesAreCreated}, {"should succeed only in creation of some resources and not all and should record error in lastErrors and lastOperation", testFailureToCreateAllResources}, {"should not reconcile spec when reconciliation is suspended", testWhenReconciliationIsSuspended}, - {"should not reconcile when no reconcile operation annotation is set", testWhenNoReconcileOperationAnnotationIsSet}, + {"should not reconcile upon etcd spec updation when no reconcile operation annotation is set", testEtcdSpecUpdateWhenNoReconcileOperationAnnotationIsSet}, + {"should reconcile upon etcd spec updation when reconcile operation annotation is set", testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet}, } g := NewWithT(t) for _, test := range tests { @@ -91,7 +95,6 @@ func testAllManagedResourcesAreCreated(t *testing.T, testNs string, reconcilerTe WithClientTLS(). WithPeerTLS(). WithReplicas(3). - WithAnnotations(map[string]string{v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile}). Build() g.Expect(etcdInstance.Spec.Backup.Store).ToNot(BeNil()) g.Expect(etcdInstance.Spec.Backup.Store.SecretRef).ToNot(BeNil()) @@ -167,7 +170,7 @@ func testWhenReconciliationIsSuspended(t *testing.T, testNs string, reconcilerTe WithReplicas(3). WithAnnotations(map[string]string{ v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "true", + druidv1alpha1.SuspendEtcdSpecReconcileAnnotation: "", }).Build() ctx := context.Background() cl := reconcilerTestEnv.itTestEnv.GetClient() @@ -181,7 +184,7 @@ func testWhenReconciliationIsSuspended(t *testing.T, testNs string, reconcilerTe assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), true, 5*time.Second, 1*time.Second) } -func testWhenNoReconcileOperationAnnotationIsSet(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { +func testEtcdSpecUpdateWhenNoReconcileOperationAnnotationIsSet(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { // ***************** setup ***************** g := NewWithT(t) etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). @@ -191,12 +194,57 @@ func testWhenNoReconcileOperationAnnotationIsSet(t *testing.T, testNs string, re Build() ctx := context.Background() cl := reconcilerTestEnv.itTestEnv.GetClient() - // create etcdInstance resource - g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + // create etcdInstance resource and assert successful reconciliation, and ensure that sts generation is 1 + createAndAssertEtcdReconciliation(ctx, t, reconcilerTestEnv, etcdInstance) + assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 1, 30*time.Second, 2*time.Second) + // get updated version of etcdInstance + g.Expect(cl.Get(ctx, etcdInstance.GetNamespaceName(), etcdInstance)).To(Succeed()) + // update etcdInstance spec without reconcile operation annotation set + metricsLevelExtensive := druidv1alpha1.Extensive + etcdInstance.Spec.Etcd = druidv1alpha1.EtcdConfig{Metrics: &metricsLevelExtensive} + g.Expect(cl.Update(ctx, etcdInstance)).To(Succeed()) + // ***************** test etcd spec reconciliation ***************** - assertNoComponentsExist(ctx, t, reconcilerTestEnv, etcdInstance, 10*time.Second, 2*time.Second) - assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), nil, 5*time.Second, 1*time.Second) - assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, cl, client.ObjectKeyFromObject(etcdInstance), nil, nil, 5*time.Second, 1*time.Second) + assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 2*time.Second, 2*time.Second) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) + // ensure that sts generation does not change, ie, it should remain 1, as sts is not updated after etcd spec change without reconcile operation annotation + assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 1, 30*time.Second, 2*time.Second) +} + +func testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { + // ***************** setup ***************** + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithoutDefaults(testutils.TestEtcdName, testNs). + WithClientTLS(). + WithPeerTLS(). + WithReplicas(3). + Build() + ctx := context.Background() + cl := reconcilerTestEnv.itTestEnv.GetClient() + // create etcdInstance resource and assert successful reconciliation, and ensure that sts generation is 1 + createAndAssertEtcdReconciliation(ctx, t, reconcilerTestEnv, etcdInstance) + assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 1, 30*time.Second, 2*time.Second) + // get latest version of etcdInstance + g.Expect(cl.Get(ctx, etcdInstance.GetNamespaceName(), etcdInstance)).To(Succeed()) + // update etcdInstance spec with reconcile operation annotation also set + metricsLevelExtensive := druidv1alpha1.Extensive + etcdInstance.Spec.Etcd = druidv1alpha1.EtcdConfig{Metrics: &metricsLevelExtensive} + etcdInstance.Annotations = map[string]string{ + v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, + } + g.Expect(cl.Update(ctx, etcdInstance)).To(Succeed()) + + // ***************** test etcd spec reconciliation ***************** + assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 30*time.Second, 2*time.Second) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(2), 30*time.Second, 1*time.Second) + expectedLastOperation := &druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: druidv1alpha1.LastOperationStateSucceeded, + } + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, nil, 5*time.Second, 1*time.Second) + assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), false, 5*time.Second, 1*time.Second) + // ensure that sts generation is updated to 2, since reconciliation of the etcd spec change causes an update of the sts spec + assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 2, 30*time.Second, 2*time.Second) } // ------------------------------ reconcile deletion tests ------------------------------ @@ -527,6 +575,21 @@ func createAndAssertEtcdAndAllManagedResources(ctx context.Context, t *testing.T t.Logf("successfully added finalizer to etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) } +func createAndAssertEtcdReconciliation(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { + // create etcd and assert etcd spec reconciliation + createAndAssertEtcdAndAllManagedResources(ctx, t, reconcilerTestEnv, etcdInstance) + + // assert etcd status reconciliation + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) + expectedLastOperation := &druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: druidv1alpha1.LastOperationStateSucceeded, + } + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, nil, 5*time.Second, 1*time.Second) + assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), false, 5*time.Second, 1*time.Second) + t.Logf("successfully reconciled status of etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) +} + func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey client.ObjectKey) { etcdInstance := &druidv1alpha1.Etcd{} g.Expect(cl.Get(ctx, etcdObjectKey, etcdInstance)).To(Succeed()) From 6bb461f2e65debfab7e825c81559a3e58b2fd142 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 14:22:42 +0530 Subject: [PATCH 176/235] Disable sentinel webhook when deploying druid via skaffold; to be enabled only via env var DRUID_ENABLE_SENTINEL_WEBHOOK=true --- skaffold.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/skaffold.yaml b/skaffold.yaml index 9b942cab9..ea8721e91 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -29,9 +29,6 @@ deploy: compaction: etcdEventsThreshold: 15 metricsScrapeWaitDuration: 30s - webhooks: - sentinel: - enabled: true profiles: - name: e2e-test activation: @@ -42,6 +39,15 @@ profiles: value: etcd: etcdStatusSyncPeriod: 5s + - name: enable-sentinel-webhook + activation: + - env: "DRUID_ENABLE_SENTINEL_WEBHOOK=true" + patches: + - op: add + path: /deploy/helm/releases/0/setValues/webhooks + value: + sentinel: + enabled: true - name: do-not-use-feature-gates activation: - env: "USE_ETCD_DRUID_FEATURE_GATES=false" From 78b9f8e84be2c70f31573980c959df74eea643dc Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 14:23:14 +0530 Subject: [PATCH 177/235] Minor cosmetic corrections --- api/v1alpha1/types_etcd_test.go | 12 ++++++------ hack/tools.mk | 2 +- internal/controller/utils/reconciler.go | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 5914383eb..aa9833891 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -155,7 +155,7 @@ var _ = Describe("Etcd", func() { Context("when etcd has only suspend-etcd-spec-reconcile annotation set", func() { It("should return suspend-etcd-spec-reconcile annotation", func() { etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "true", + SuspendEtcdSpecReconcileAnnotation: "", } Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) }) @@ -163,8 +163,8 @@ var _ = Describe("Etcd", func() { Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { It("should return suspend-etcd-spec-reconcile annotation", func() { etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "true", - IgnoreReconciliationAnnotation: "true", + SuspendEtcdSpecReconcileAnnotation: "", + IgnoreReconciliationAnnotation: "", } Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) }) @@ -181,7 +181,7 @@ var _ = Describe("Etcd", func() { Context("when etcd has only suspend-etcd-spec-reconcile annotation set", func() { It("should return true", func() { etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "true", + SuspendEtcdSpecReconcileAnnotation: "", } Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) @@ -197,8 +197,8 @@ var _ = Describe("Etcd", func() { Context("when etcd has both suspend-etcd-spec-reconcile and ignore-reconcile annotations set", func() { It("should return true", func() { etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "true", - IgnoreReconciliationAnnotation: "true", + SuspendEtcdSpecReconcileAnnotation: "", + IgnoreReconciliationAnnotation: "", } Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) }) diff --git a/hack/tools.mk b/hack/tools.mk index a855bbd80..3ff56ed8b 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -26,4 +26,4 @@ $(KUSTOMIZE): @test -s $(TOOLS_BIN_DIR)/kustomize || GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install sigs.k8s.io/kustomize/kustomize/v4@${KUSTOMIZE_VERSION} $(GOTESTFMT): - GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@$(GOTESTFMT_VERSION) \ No newline at end of file + GOBIN=$(abspath $(TOOLS_BIN_DIR)) go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@$(GOTESTFMT_VERSION) diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index a007d0446..db1a278ad 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -6,7 +6,6 @@ package utils import ( "context" - _ "embed" "errors" "fmt" "time" From b9e9cefaaef7d44f3c5aab0b17943b0058f27481 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 15:26:17 +0530 Subject: [PATCH 178/235] Address review comments by @anveshreddy18 --- docs/concepts/controllers.md | 8 ++++---- ...recovery-from-permanent-quorum-loss-in-etcd-cluster.md | 2 +- internal/operator/snapshotlease/snapshotlease.go | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/concepts/controllers.md b/docs/concepts/controllers.md index 60fece673..f4d53f60e 100644 --- a/docs/concepts/controllers.md +++ b/docs/concepts/controllers.md @@ -46,7 +46,7 @@ The *etcd controller* is responsible for the reconciliation of the `Etcd` resour Additionally, *etcd controller* also periodically updates the `Etcd` resource status with the latest available information from the etcd cluster, as well as results and errors from the recent-most reconciliation of the `Etcd` resource spec. -The *etcd controller* is essential to the functioning of the etcd cluster and etcd-druid, thus the minimum number of worker threads is 1 (default being 3). +The *etcd controller* is essential to the functioning of the etcd cluster and etcd-druid, thus the minimum number of worker threads is 1 (default being 3), controlled by the CLI flag `--etcd-workers`. ### `Etcd` Spec Reconciliation @@ -87,7 +87,7 @@ The controller watches the number of events accumulated as part of delta snapsho The controller watches for changes in *snapshot* `Leases` associated with `Etcd` resources. It checks the full and delta snapshot `Leases` and calculates the difference in events between the latest delta snapshot and the previous full snapshot, and initiates the compaction job if the event threshold is crossed. -The number of worker threads for the *compaction controller* needs to be greater than or equal to 0 (default 3). +The number of worker threads for the *compaction controller* needs to be greater than or equal to 0 (default 3), controlled by the CLI flag `--compaction-workers`. This is unlike other controllers which need at least one worker thread for the proper functioning of etcd-druid as snapshot compaction is not a core functionality for the etcd clusters to be deployed. The compaction controller should be explicitly enabled by the user, through the `--enable-backup-compaction` CLI flag. @@ -96,7 +96,7 @@ The compaction controller should be explicitly enabled by the user, through the The *etcdcopybackupstask controller* is responsible for deploying the [`etcdbrctl copy`](https://github.com/gardener/etcd-backup-restore/blob/master/cmd/copy.go) command as a job. This controller reacts to create/update events arising from EtcdCopyBackupsTask resources, and deploys the `EtcdCopyBackupsTask` job with source and target backup storage providers as arguments, which are derived from source and target bucket secrets referenced by the `EtcdCopyBackupsTask` resource. -The number of worker threads for the *etcdcopybackupstask controller* needs to be greater than or equal to 0 (default being 3). +The number of worker threads for the *etcdcopybackupstask controller* needs to be greater than or equal to 0 (default being 3), controlled by the CLI flag `--etcd-copy-backups-task-workers`. This is unlike other controllers who need at least one worker thread for the proper functioning of etcd-druid as `EtcdCopyBackupsTask` is not a core functionality for the etcd clusters to be deployed. ## Secret Controller @@ -107,4 +107,4 @@ This finalizer is added to ensure that `Secret`s which are referenced by the `Et Events arising from the `Etcd` resource are mapped to a list of `Secret`s such as backup and TLS secrets that are referenced by the `Etcd` resource, and are enqueued into the request queue, which the reconciler then acts on. -The number of worker threads for the secret controller must be at least 1 (default being 10) for this core controller, since the referenced TLS and infrastructure access secrets are essential to the proper functioning of the etcd cluster. +The number of worker threads for the secret controller must be at least 1 (default being 10) for this core controller, controlled by the CLI flag `--secret-workers`, since the referenced TLS and infrastructure access secrets are essential to the proper functioning of the etcd cluster. diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md index 9324126f2..734baaa0b 100644 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -1,7 +1,7 @@ # Recovery from Permanent Quorum Loss in an Etcd Cluster ## Quorum loss in Etcd Cluster -[Quorum loss](https://etcd.io/docs/v3.4/op-guide/recovery/) means when the majority of Etcd pods(greater than or equal to n/2 + 1) are down simultaneously for some reason. +[Quorum loss](https://etcd.io/docs/v3.4/op-guide/recovery/) means when the majority of Etcd pods (greater than or equal to n/2 + 1) are down simultaneously for some reason. There are two types of quorum loss that can happen to [Etcd multinode cluster](https://github.com/gardener/etcd-druid/tree/master/docs/proposals/multi-node) : diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/operator/snapshotlease/snapshotlease.go index 6d233f2fd..ef5a27eb3 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/operator/snapshotlease/snapshotlease.go @@ -78,6 +78,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd } // Sync creates or updates the snapshot leases for the given Etcd. +// If backups are disabled for the Etcd resource, any existing snapshot leases are deleted. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { ctx.Logger.Info("Backup has been disabled. Triggering deletion of snapshot leases") From edb735ab6f1ef0aa2d11270f36d20eacf7b117e8 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 16:12:36 +0530 Subject: [PATCH 179/235] Address review comments by @seshachalam-yv; restructured Makefile --- Makefile | 163 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 87 insertions(+), 76 deletions(-) diff --git a/Makefile b/Makefile index 16d7ac1af..35c627996 100644 --- a/Makefile +++ b/Makefile @@ -25,70 +25,34 @@ TOOLS_DIR := $(HACK_DIR)/tools include $(GARDENER_HACK_DIR)/tools.mk include $(HACK_DIR)/tools.mk +##################################################################### +# Rules for verification, formatting, linting, testing and cleaning # +##################################################################### + .PHONY: tidy tidy: @env GO111MODULE=on go mod tidy @GARDENER_HACK_DIR=$(GARDENER_HACK_DIR) bash $(HACK_DIR)/update-github-templates.sh @cp $(GARDENER_HACK_DIR)/cherry-pick-pull.sh $(HACK_DIR)/cherry-pick-pull.sh && chmod +xw $(HACK_DIR)/cherry-pick-pull.sh -kind-up kind-down ci-e2e-kind deploy-localstack test-e2e: export KUBECONFIG = $(KUBECONFIG_PATH) - -all: druid - -# Build manager binary -.PHONY: druid -druid: fmt check - @env GO111MODULE=on go build -o bin/druid main.go - -# Run against the configured Kubernetes cluster in ~/.kube/config -.PHONY: run -run: - go run ./main.go - -# Install CRDs into a cluster -.PHONY: install -install: manifests - kubectl apply -f config/crd/bases - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -.PHONY: deploy-via-kustomize -deploy-via-kustomize: manifests $(KUSTOMIZE) - kubectl apply -f config/crd/bases - kustomize build config/default | kubectl apply -f - - -# Deploy controller to the Kubernetes cluster specified in the environment variable KUBECONFIG -# Modify the Helm template located at charts/druid/templates if any changes are required -.PHONY: deploy -deploy: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) run -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) - -.PHONY: deploy-dev -deploy-dev: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) dev -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) - -.PHONY: deploy-debug -deploy-debug: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) debug -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) +.PHONY: clean +clean: + @bash $(GARDENER_HACK_DIR)/clean.sh ./api/... ./internal/... -.PHONY: undeploy -undeploy: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) delete -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) +.PHONY: update-dependencies +update-dependencies: + @env GO111MODULE=on go get -u + @make tidy -# Generate manifests e.g. CRD, RBAC etc. -.PHONY: manifests -manifests: $(VGOPATH) $(CONTROLLER_GEN) - @GARDENER_HACK_DIR=$(GARDENER_HACK_DIR) VGOPATH=$(VGOPATH) go generate ./config/crd/bases - @find "$(REPO_ROOT)/config/crd/bases" -name "*.yaml" -exec cp '{}' "$(REPO_ROOT)/charts/druid/charts/crds/templates/" \; - @controller-gen rbac:roleName=manager-role paths="./internal/controller/..." +.PHONY: add-license-headers +add-license-headers: $(GO_ADD_LICENSE) + @bash $(HACK_DIR)/addlicenseheaders.sh ${YEAR} # Run go fmt against code .PHONY: fmt fmt: @env GO111MODULE=on go fmt ./... -clean: - @bash $(GARDENER_HACK_DIR)/clean.sh ./api/... ./internal/... - # Check packages .PHONY: check check: $(GOLANGCI_LINT) $(GOIMPORTS) fmt manifests @@ -98,24 +62,19 @@ check: $(GOLANGCI_LINT) $(GOIMPORTS) fmt manifests check-generate: @bash $(GARDENER_HACK_DIR)/check-generate.sh "$(REPO_ROOT)" +# Generate manifests e.g. CRD, RBAC etc. +.PHONY: manifests +manifests: $(VGOPATH) $(CONTROLLER_GEN) + @GARDENER_HACK_DIR=$(GARDENER_HACK_DIR) VGOPATH=$(VGOPATH) go generate ./config/crd/bases + @find "$(REPO_ROOT)/config/crd/bases" -name "*.yaml" -exec cp '{}' "$(REPO_ROOT)/charts/druid/charts/crds/templates/" \; + @controller-gen rbac:roleName=manager-role paths="./internal/controller/..." + # Generate code .PHONY: generate generate: manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) @go generate "$(REPO_ROOT)/internal/..." @"$(HACK_DIR)/update-codegen.sh" -# Build the docker image -.PHONY: docker-build -docker-build: - docker build . -t ${IMG} --rm - @echo "updating kustomize image patch file for manager resource" - sed -i'' -e 's@image: .*@image: '"${IMG}"'@' ./config/default/manager_image_patch.yaml - -# Push the docker image -.PHONY: docker-push -docker-push: - docker push ${IMG} - # Run tests .PHONY: test test: $(GINKGO) $(GOTESTFMT) @@ -130,6 +89,11 @@ test: $(GINKGO) $(GOTESTFMT) # run the golang native unit tests. @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... +.PHONY: test-integration +test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) + @SETUP_ENVTEST="true" "$(HACK_DIR)/test.sh" ./test/integration/... + @SETUP_ENVTEST="true" "$(HACK_DIR)/test-go.sh" ./test/it/... + .PHONY: test-cov test-cov: $(GINKGO) $(SETUP_ENVTEST) @TEST_COV="true" bash $(HACK_DIR)/test.sh --skip-package=./test/e2e @@ -138,23 +102,32 @@ test-cov: $(GINKGO) $(SETUP_ENVTEST) test-cov-clean: @bash $(GARDENER_HACK_DIR)/test-cover-clean.sh -.PHONY: test-e2e -test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) - @bash $(HACK_DIR)/e2e-test/run-e2e-test.sh $(PROVIDERS) +################################################################# +# Rules related to binary build, Docker image build and release # +################################################################# -.PHONY: test-integration -test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) - @SETUP_ENVTEST="true" "$(HACK_DIR)/test.sh" ./test/integration/... - @SETUP_ENVTEST="true" "$(HACK_DIR)/test-go.sh" ./test/it/... +# Build manager binary +.PHONY: druid +druid: fmt check + @env GO111MODULE=on go build -o bin/druid main.go -.PHONY: update-dependencies -update-dependencies: - @env GO111MODULE=on go get -u - @make tidy +# Build the docker image +.PHONY: docker-build +docker-build: + docker build . -t ${IMG} --rm + @echo "updating kustomize image patch file for manager resource" + sed -i'' -e 's@image: .*@image: '"${IMG}"'@' ./config/default/manager_image_patch.yaml -.PHONY: add-license-headers -add-license-headers: $(GO_ADD_LICENSE) - @bash $(HACK_DIR)/addlicenseheaders.sh ${YEAR} +# Push the docker image +.PHONY: docker-push +docker-push: + docker push ${IMG} + +##################################################################### +# Rules for local environment # +##################################################################### + +kind-up kind-down ci-e2e-kind deploy-localstack test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH) .PHONY: kind-up kind-up: $(KIND) @@ -165,10 +138,48 @@ kind-up: $(KIND) kind-down: $(KIND) $(KIND) delete cluster --name etcd-druid-e2e +# Install CRDs into a cluster +.PHONY: install +install: manifests + kubectl apply -f config/crd/bases + +# Run against the configured Kubernetes cluster in ~/.kube/config or specified by environment variable KUBECONFIG +.PHONY: run +run: + go run ./main.go + +# Deploy controller in the configured Kubernetes cluster in ~/.kube/config +.PHONY: deploy-via-kustomize +deploy-via-kustomize: manifests $(KUSTOMIZE) + kubectl apply -f config/crd/bases + kustomize build config/default | kubectl apply -f - + +# Deploy controller to the Kubernetes cluster specified in the environment variable KUBECONFIG +# Modify the Helm template located at charts/druid/templates if any changes are required +.PHONY: deploy +deploy: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) run -m etcd-druid + +.PHONY: deploy-dev +deploy-dev: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) dev -m etcd-druid + +.PHONY: deploy-debug +deploy-debug: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) debug -m etcd-druid + +.PHONY: undeploy +undeploy: $(SKAFFOLD) $(HELM) + $(SKAFFOLD) delete -m etcd-druid + .PHONY: deploy-localstack deploy-localstack: $(KUBECTL) @bash $(HACK_DIR)/deploy-localstack.sh +.PHONY: test-e2e +test-e2e: $(KUBECTL) $(HELM) $(SKAFFOLD) $(KUSTOMIZE) + @bash $(HACK_DIR)/e2e-test/run-e2e-test.sh $(PROVIDERS) + .PHONY: ci-e2e-kind ci-e2e-kind: @BUCKET_NAME=$(BUCKET_NAME) bash $(HACK_DIR)/ci-e2e-kind.sh From f75c718cfa6b966328bd6b37843ab1d69d6e9a56 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 19:32:41 +0530 Subject: [PATCH 180/235] Address review comments by @seshachalam-yv; Sentinel webhook now also handles sts/scale subresource --- .../templates/validating-webhook-config.yaml | 27 ++++++++++++++ internal/webhook/sentinel/handler.go | 37 ++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index c9781394a..301ff95b4 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -90,4 +90,31 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 10 + - admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3ekNDQWRlZ0F3SUJBZ0lKQU51a3lQUVVxYnhNTUEwR0NTcUdTSWIzRFFFQkN3VUFNQlV4RXpBUkJnTlYKQkFNTUNtVjBZMlF0WkhKMWFXUXdIaGNOTWpRd01URTRNVFV6TVRJeldoY05NelF3TVRFNE1UVXpNVEl6V2pBVgpNUk13RVFZRFZRUUREQXBsZEdOa0xXUnlkV2xrTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCCkNnS0NBUUVBeWx1TFQveTl1YmxVdVJ0NDBOUW1xRTVFSWxlakROVTRnREd3WDJ3K2t6RHZ0bFdsYlF0STdQSVoKWTZSK1kvYXF2OHhiOCt3R0tlWEM4bGtSQ1BsZjNIdGpTVUxWUkkzYnVBd29WODJ2YjU4NlQ5Z0Q3OHJKVHdYVwpxcm1kc25VUGR5UzAwSlFQY2d3TzE3QW9DSXl0MkVyclQzWlpyMFpsMVJsUWZoTTc0VVpNbi8xeTJqNUFmUTJPCmh2emdmYnl6ZHZNT0hycnd4NHoxblBsVjNPU3M3WGo3TkM5UDZ4bnBTVTU1MnF1bkZmTDdUc3U3OTQ4NWdZZ3AKOFYrTG9oclY0MWdGWmNlVlh6Y3k1TGhGK093eFBGSG0rMlJlWTZ2UmxmMlU5S2lTTzA2MVBOcy9DYklhWDZucwpmMC9oTVJYbm16c0Q2VU1zd0ZxT0Q4UWw3VTUzQ1FJREFRQUJvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBYVl3CkR3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXFWUG93RXR4akFvT0xtOVl6ZUhIZzBFMmdWb3cKRFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUR3ZGduT25POStWNmNvdTQ4Z1czeUtONVhMNmJkRm5mbng5aldlTQpkelBsc2tBa0hKTzAvTXpqbk1DUnFxQmREajM2Yzcvb3lJcEpBall1ZmFrMXJDcW9rVEpHOVNzTlE4VlJmT0FSCkRRU0o0YmsrMzNQT29MSnNrZTNzdm9ZM0RPc3h4RzZNT0o5S29aRTFaeWxEd3hrYVo0U212b1NmTStmUHU3aloKenA1RWJXVnE0YmJUYjgrRnZqSDQ5bmY1eGY1WHJLbTJ1Z3dZZWIwWnpnYkIrSGdNSHRvMGtWcXFObVlrbDYyMwp0NFBOV3Q1ck9sY2swd1dKeVVZZHdONk1qQWQrZVZaQmJRRzQ3TWpZNTJTenBwbGNQdVJuaU81Q0tKb0JZdmU1CmZzTUhIR0dXeEg0b2JCNk9WZytIM0pJNGpRWndTUXZGRVBKcm5LNXV0RjdySmw0PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: etcd-druid + namespace: {{ .Release.Namespace }} + path: /webhooks/sentinel + port: {{ .Values.controllerManager.server.webhook.port }} + failurePolicy: Fail + matchPolicy: Exact + name: stsscale.sentinel.webhooks.druid.gardener.cloud + namespaceSelector: {} + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - statefulsets/scale + scope: '*' + sideEffects: None + timeoutSeconds: 10 {{- end }} diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 20b6134b8..741324cbe 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/scale/scheme/autoscalingv1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -60,8 +61,9 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) } - // Leases (member and snapshot) will be periodically updated by etcd members. Allow updates to leases. requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} + + // Leases (member and snapshot) will be periodically updated by etcd members. Allow updates to leases. if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && req.Operation == admissionv1.Update { return admission.Allowed("lease resource can be freely updated") @@ -75,6 +77,26 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Allowed(fmt.Sprintf("unexpected resource type: %s", requestGKString)) } + // If admission request is for statefulsets/scale subresource, then check labels on the parent statefulset object + // and allow the request if it doesn't contain label `app.kubernetes.io/managed-by: etcd-druid`. + // This special handling is required for statefulsets/scale subresource, since it does not work when + // an objectSelector is specified for it in the ValidatingWebhookConfiguration. + // More information can be found at https://github.com/kubernetes/kubernetes/issues/113594#issuecomment-1332573990 + requestResourceGK := schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} + if requestGK == autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind() && + requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() { + sts, err := h.fetchStatefulSet(ctx, req.Name, req.Namespace) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + managedBy, hasLabel := sts.GetLabels()[druidv1alpha1.LabelManagedByKey] + if !hasLabel || managedBy != druidv1alpha1.LabelManagedByValue { + return admission.Allowed(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey)) + } + // replicate sts labels to the /scale subresource object, used for subsequent checks + obj.SetLabels(sts.GetLabels()) + } + etcdName, hasLabel := obj.GetLabels()[druidv1alpha1.LabelPartOfKey] if !hasLabel { return admission.Allowed(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey)) @@ -124,6 +146,7 @@ func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.Gr rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(), rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(), appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), + autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(), // for statefulsets' /scale subresource policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): @@ -150,3 +173,15 @@ func (h *Handler) doDecodeRequestObject(req admission.Request) (client.Object, e } return obj, nil } + +func (h *Handler) fetchStatefulSet(ctx context.Context, name, namespace string) (*appsv1.StatefulSet, error) { + var ( + err error + sts = &appsv1.StatefulSet{} + ) + + if err = h.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, sts); err != nil { + return nil, err + } + return sts, nil +} From 318326d452a8ea7cfb08c25ab9acf0e029d7ee80 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 15 May 2024 19:35:46 +0530 Subject: [PATCH 181/235] Enable sentinel webhook for CI e2e tests, run via `make ci-e2e-kind` --- hack/ci-e2e-kind.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/ci-e2e-kind.sh b/hack/ci-e2e-kind.sh index b3096fbb3..f4acb6bc1 100755 --- a/hack/ci-e2e-kind.sh +++ b/hack/ci-e2e-kind.sh @@ -25,5 +25,6 @@ make LOCALSTACK_HOST="localstack.default:4566" \ AWS_REGION="us-east-2" \ PROVIDERS="aws" \ TEST_ID="$BUCKET_NAME" \ + DRUID_ENABLE_SENTINEL_WEBHOOK=true \ STEPS="setup,deploy,test" \ test-e2e \ No newline at end of file From 7b23d257c5e8cc782ab191fc5d499a370032f5d1 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 16 May 2024 11:35:57 +0530 Subject: [PATCH 182/235] Address review comments by @seshachalam-yv; update documentation for new changes --- docs/concepts/controllers.md | 4 +- docs/development/getting-started-locally.md | 46 +++++++++++++-------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/docs/concepts/controllers.md b/docs/concepts/controllers.md index f4d53f60e..3995b0984 100644 --- a/docs/concepts/controllers.md +++ b/docs/concepts/controllers.md @@ -50,7 +50,9 @@ The *etcd controller* is essential to the functioning of the etcd cluster and et ### `Etcd` Spec Reconciliation -While building the controller, an event filter is set such that the behavior of the controller depends on the `gardener.cloud/operation: reconcile` *annotation*. This is controlled by the `--enable-etcd-spec-auto-reconcile` CLI flag, which, if set to `false`, tells the controller to perform reconciliation only when this annotation is present. If the flag is set to `true`, the controller will reconcile the etcd cluster anytime the `Etcd` spec, and thus `generation`, changes, and the next queued event for it is triggered. +While building the controller, an event filter is set such that the behavior of the controller, specifically for `Etcd` update operations, depends on the `gardener.cloud/operation: reconcile` *annotation*. This is controlled by the `--enable-etcd-spec-auto-reconcile` CLI flag, which, if set to `false`, tells the controller to perform reconciliation only when this annotation is present. If the flag is set to `true`, the controller will reconcile the etcd cluster anytime the `Etcd` spec, and thus `generation`, changes, and the next queued event for it is triggered. + +> **Note:** Creation and deletion of `Etcd` resources are not affected by the above flag or annotation. The reason this filter is present is that any disruption in the `Etcd` resource due to reconciliation (due to changes in the `Etcd` spec, for example) while workloads are being run would cause unwanted downtimes to the etcd cluster. Hence, any user who wishes to avoid such disruptions, can choose to set the `--enable-etcd-spec-auto-reconcile` CLI flag to `false`. An example of this is Gardener's [gardenlet](https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md), which reconciles the `Etcd` resource only during a shoot cluster's [*maintenance window*](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_maintenance.md). diff --git a/docs/development/getting-started-locally.md b/docs/development/getting-started-locally.md index 4b441d38f..287910953 100644 --- a/docs/development/getting-started-locally.md +++ b/docs/development/getting-started-locally.md @@ -36,11 +36,24 @@ export KUBECONFIG=$PWD/hack/e2e-test/infrastructure/kind/kubeconfig ## Setting up etcd-druid -```sh -make deploy -``` +Either one of these commands may be used to deploy etcd-druid to the configured k8s cluster. + +1. The following command deploys etcd-druid to the configured k8s cluster: + ```sh + make deploy + ``` + +2. The following command deploys etcd-druid to the configured k8s cluster using Skaffold `dev` mode, such that changes in the etcd-druid code are automatically picked up and applied to the deployment. This helps with local development and quick iterative changes: + ```sh + make deploy-dev + ``` + +3. The following command deploys etcd-druid to the configured k8s cluster using Skaffold `debug` mode, so that a debugger can be attached to the running etcd-druid deployment. Please refer to [this guide](https://skaffold.dev/docs/workflows/debug/) for more information on Skaffold-based debugging: + ```sh + make deploy-debug + ``` -This generates the `Etcd` CRD and deploys an etcd-druid pod into the cluster +This generates the `Etcd` and `EtcdCopyBackupsTask` CRDs and deploys an etcd-druid pod into the cluster. ### Prepare the Etcd CR @@ -50,7 +63,7 @@ The Etcd CR can be found at this location `$PWD/config/samples/druid_v1alpha1_et - **Without Backups enabled** - To setup Etcd-druid without backups enabled, make sure the `spec.backup.store` section of the Etcd CR is commented out. + To set up etcd-druid without backups enabled, make sure the `spec.backup.store` section of the Etcd CR is commented out. - **With Backups enabled (On Cloud Provider Object Stores)** @@ -102,19 +115,6 @@ Create the Etcd CR (Custom Resource) by applying the Etcd yaml to the cluster kubectl apply -f config/samples/druid_v1alpha1_etcd.yaml ``` -### Annotate Etcd CR with the reconcile annotation - -> **Note :** If the `--enable-etcd-spec-auto-reconcile` flag is set to `true`, this step is not required. - -The above step creates an Etcd resource, however etcd-druid won't pick it up for reconciliation without an annotation. To get etcd-druid to reconcile the etcd CR, annotate it with the following `gardener.cloud/operation: reconcile`. - -```sh -# Annotate etcd-test CR to reconcile -kubectl annotate etcd etcd-test gardener.cloud/operation="reconcile" -``` - -This starts creating the etcd cluster - ### Verify the Etcd cluster To obtain information regarding the newly instantiated etcd cluster, perform the following step, which gives details such as the cluster size, readiness status of its members, and various other attributes. @@ -150,6 +150,16 @@ For a multi-node etcd cluster, insert the key-value pair from the `etcd` contain The Etcd database file is located at `var/etcd/data/new.etcd/snap/db` inside the `backup-restore` container. In versions with an `alpine` base image, you can exec directly into the container. However, in recent versions where the `backup-restore` docker image started using a distroless image, a debug container is required to communicate with it, as mentioned in the previous section. +### Updating the Etcd CR + +The `Etcd` spec can be updated with new changes, such as etcd cluster configuration or backup-restore configuration, and etcd-druid will reconcile these changes as expected, under certain conditions: +1. If the `--enable-etcd-spec-auto-reconcile` flag is set to `true`, the spec change is automatically picked up and reconciled by etcd-druid. +2. If the `--enable-etcd-spec-auto-reconcile` flag is unset, or set to `false`, then etcd-druid will expect an additional annotation `gardener.cloud/operation: reconcile` on the `Etcd` resource in order to pick it up for reconciliation. Upon successful reconciliation, this annotation is removed by etcd-druid. The annotation can be added as follows: + ```sh + # Annotate etcd-test CR to reconcile + kubectl annotate etcd etcd-test gardener.cloud/operation="reconcile" + ``` + ## Cleaning the setup ```sh From e3f542911ba930b1c8da2e0d1b692d21a40c1982 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 16 May 2024 21:54:58 +0530 Subject: [PATCH 183/235] Helm will create validating webhook config resource only if atleast one webhook is enabled --- charts/druid/templates/validating-webhook-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index 301ff95b4..37ab73686 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -1,3 +1,9 @@ +{{- $shouldCreateVWC := false }} +{{- range $webhookName, $webhookConfig := .Values.webhooks }} +{{- $webhookEnabled := ($webhookConfig).enabled | default false }} +{{- $shouldCreateVWC = or $shouldCreateVWC $webhookEnabled }} +{{- end }} +{{- if $shouldCreateVWC }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration @@ -118,3 +124,4 @@ webhooks: sideEffects: None timeoutSeconds: 10 {{- end }} +{{- end }} From 53d5fef76966e9ac25beb675dd667319b8f1fbdc Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 17 May 2024 07:55:47 +0530 Subject: [PATCH 184/235] Address review comments by @ishan16696: update docstrings, flag descriptions --- api/v1alpha1/types_etcd.go | 2 +- docs/deployment/cli-flags.md | 4 ++-- internal/controller/etcd/config.go | 6 ++++-- internal/controller/utils/etcdstatus.go | 3 +++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 22ea655ca..58c653b95 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -597,7 +597,7 @@ func (e *Etcd) IsSpecReconciliationSuspended() bool { return suspendReconcileAnnotKey != nil } -// AreManagedResourcesProtected returns false if the Etcd resource has the disable-resource-protection annotation set, +// AreManagedResourcesProtected returns false if the Etcd resource has the `druid.gardener.cloud/disable-resource-protection` annotation set, // else returns true. func (e *Etcd) AreManagedResourcesProtected() bool { return !metav1.HasAnnotation(e.ObjectMeta, DisableResourceProtectionAnnotation) diff --git a/docs/deployment/cli-flags.md b/docs/deployment/cli-flags.md index 526b57bff..2da5546c0 100644 --- a/docs/deployment/cli-flags.md +++ b/docs/deployment/cli-flags.md @@ -16,8 +16,8 @@ Etcd-druid exposes the following CLI flags that allow for configuring its behavi | `leader-election-resource-lock` | `controller-manager` | Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.
Deprecated. Will be removed in the future in favour of using only `leases` as the leader election resource lock for the controller manager. | `"leases"` | | `disable-lease-cache` | `controller-manager` | Disable cache for lease.coordination.k8s.io resources. | `false` | | `etcd-workers` | `etcd-controller` | Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed. | `3` | -| `ignore-operation-annotation` | `etcd-controller` | Specifies whether to ignore or honour the operation annotation on resources to be reconciled.
Deprecated. Use `--enable-etcd-spec-auto-reconcile` instead. | `false` | -| `enable-etcd-spec-auto-reconcile` | `etcd-controller` | If true then automatically reconciles Etcd Spec. If false waits for explicit annotation to be placed on the Etcd resource to trigger reconcile. | `false` | +| `ignore-operation-annotation` | `etcd-controller` | Specifies whether to ignore or honour the annotation `gardener.cloud/operation: reconcile` on resources to be reconciled.
Deprecated: please use `--enable-etcd-spec-auto-reconcile` instead. | `false` | +| `enable-etcd-spec-auto-reconcile` | `etcd-controller` | If true then automatically reconciles Etcd Spec. If false, waits for explicit annotation `gardener.cloud/operation: reconcile` to be placed on the Etcd resource to trigger reconcile. | `false` | | `disable-etcd-serviceaccount-automount` | `etcd-controller` | If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets. | `false` | | `etcd-status-sync-period` | `etcd-controller` | Period after which an etcd status sync will be attempted. | `15s` | | `etcd-member-notready-threshold` | `etcd-controller` | Threshold after which an etcd member is considered not ready if the status was unknown before. | `5m` | diff --git a/internal/controller/etcd/config.go b/internal/controller/etcd/config.go index 62f805333..93cd74d8b 100644 --- a/internal/controller/etcd/config.go +++ b/internal/controller/etcd/config.go @@ -6,10 +6,12 @@ package etcd import ( "errors" + "fmt" "time" "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" + gardenerconstants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" flag "github.com/spf13/pflag" "k8s.io/component-base/featuregate" ) @@ -79,9 +81,9 @@ func InitFromFlags(fs *flag.FlagSet, cfg *Config) { fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, "Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed.") flag.BoolVar(&cfg.IgnoreOperationAnnotation, ignoreOperationAnnotationFlagName, defaultIgnoreOperationAnnotation, - "Specifies whether to ignore or honour the operation annotation on resources to be reconciled.") + fmt.Sprintf("Specifies whether to ignore or honour the annotation `%s: %s` on resources to be reconciled. Deprecated: please use `--%s` instead.", gardenerconstants.GardenerOperation, gardenerconstants.GardenerOperationReconcile, enableEtcdSpecAutoReconcileFlagName)) flag.BoolVar(&cfg.EnableEtcdSpecAutoReconcile, enableEtcdSpecAutoReconcileFlagName, defaultEnableEtcdSpecAutoReconcile, - "If true then automatically reconciles Etcd Spec. If false waits for explicit annotation to be placed on the Etcd resource to trigger reconcile.") + fmt.Sprintf("If true then automatically reconciles Etcd Spec. If false, waits for explicit annotation `%s: %s` to be placed on the Etcd resource to trigger reconcile.", gardenerconstants.GardenerOperation, gardenerconstants.GardenerOperationReconcile)) fs.BoolVar(&cfg.DisableEtcdServiceAccountAutomount, disableEtcdServiceAccountAutomountFlagName, defaultDisableEtcdServiceAccountAutomount, "If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets.") fs.DurationVar(&cfg.EtcdStatusSyncPeriod, etcdStatusSyncPeriodFlagName, defaultEtcdStatusSyncPeriod, diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index 274777062..75a5f7819 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -17,8 +17,11 @@ import ( // LastOperationErrorRecorder records etcd.Status.LastOperation and etcd.Status.LastErrors type LastOperationErrorRecorder interface { + // RecordStart records the start of an operation in the Etcd status RecordStart(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error + // RecordSuccess records the success of an operation in the Etcd status RecordSuccess(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error + // RecordError records an error encountered in the last operation in the Etcd status RecordError(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error } From b9978b4e77ab5ee782f2badbea21c8364f19c947 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 17 May 2024 08:44:03 +0530 Subject: [PATCH 185/235] Address review comments by @seshachalam-yv; sentinel webhook allows lease updates only from etcd members --- docs/concepts/webhooks.md | 4 ++-- internal/webhook/sentinel/handler.go | 32 +++++++++++++++++----------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/concepts/webhooks.md b/docs/concepts/webhooks.md index 331dcbc09..fb6afa4e9 100644 --- a/docs/concepts/webhooks.md +++ b/docs/concepts/webhooks.md @@ -27,10 +27,10 @@ Druid controller-manager registers and runs the [etcd controller](controllers.md Unintended changes to any of these *managed resources* can lead to misconfiguration of the etcd cluster, leading to unwanted downtime for etcd traffic. To prevent such unintended changes, a validating webhook called *sentinel webhook* guards these managed resources, ensuring that only authorized entities can perform operations on these managed resources. -*Sentinel webhook* prevents *UPDATE* and *DELETE* operations on all resources managed by *etcd controller*, unless such an operation is performed by druid itself, and during reconciliation of the `Etcd` resource. Operations are also allowed if performed by one of the authorized entities specified by CLI flag `--sentinel-exempt-service-accounts`. +*Sentinel webhook* prevents *UPDATE* and *DELETE* operations on all resources managed by *etcd controller*, unless such an operation is performed by druid itself, and during reconciliation of the `Etcd` resource. Operations are also allowed if performed by one of the authorized entities specified by CLI flag `--sentinel-exempt-service-accounts`, but only if the `Etcd` resource is not being reconciled by etcd-druid at that time. There may be specific cases where a human operator may need to make changes to the managed resources, possibly to test or fix an etcd cluster. An example of this is [recovery from permanent quorum loss](../operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md), where a human operator will need to suspend reconciliation of the `Etcd` resource, make changes to the underlying managed resources such as `StatefulSet` and `ConfigMap`, and then resume reconciliation for the `Etcd` resource. Such manual interventions will require out-of-band changes to the managed resources. Protection of managed resources for such `Etcd` resources can be turned off by adding an annotation `druid.gardener.cloud/disable-resource-protection` on the `Etcd` resource. This will effectively disable *sentinel webhook* protection for all managed resources for the specific `Etcd`. -**Note:** *UPDATE* operations for `Lease`s are always allowed, since these are regularly updated by the etcd-backup-restore sidecar. +**Note:** *UPDATE* operations for `Lease`s by etcd members are always allowed, since these are regularly updated by the etcd-backup-restore sidecar. The *sentinel webhook* is disabled by default, and can be enabled via the CLI flag `--enable-sentinel-webhook`. diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 741324cbe..2f5bbef00 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" "slices" + "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -63,12 +64,6 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} - // Leases (member and snapshot) will be periodically updated by etcd members. Allow updates to leases. - if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && - req.Operation == admissionv1.Update { - return admission.Allowed("lease resource can be freely updated") - } - obj, err := h.decodeRequestObject(req, requestGK) if err != nil { return admission.Errored(http.StatusInternalServerError, err) @@ -110,29 +105,42 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } + // Leases (member and snapshot) will be periodically updated by etcd members. + // Allow updates to such leases, but only by etcd members, which would use the serviceaccount deployed by druid for them. + if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && + req.Operation == admissionv1.Update { + saTokens := strings.Split(req.UserInfo.Username, ":") + saName := saTokens[len(saTokens)-1] // last element of sa name which looks like `system:serviceaccount::` + if saName == etcd.GetServiceAccountName() { + return admission.Allowed("lease resource can be freely updated by etcd members") + } + } + // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set if !etcd.AreManagedResourcesProtected() { return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid, - // and allow exempt service accounts to make changes to resources, but only if the Etcd is not currently being reconciled. if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { return admission.Allowed(fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", etcd.Name)) } return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", etcd.Name)) - } else { - for _, sa := range h.config.ExemptServiceAccounts { - if req.UserInfo.Username == sa { - return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) - } + } + + // allow exempt service accounts to make changes to resources, but only if the Etcd is not currently being reconciled. + for _, sa := range h.config.ExemptServiceAccounts { + if req.UserInfo.Username == sa { + return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) } } return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", etcd.Name)) } +// decodeRequestObject decodes the relevant object from the admission request and returns it. +// If it encounters an unexpected resource type, it returns a nil object. func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.GroupKind) (client.Object, error) { var ( obj client.Object From d41abe1f53e2763592226b9dceedd279f75f2881 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 17 May 2024 09:02:12 +0530 Subject: [PATCH 186/235] Address review comments by @anveshreddy18: add package-level comment for internal/controller/predicate --- internal/controller/predicate/predicate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/controller/predicate/predicate.go b/internal/controller/predicate/predicate.go index d82b885ad..fe666e314 100644 --- a/internal/controller/predicate/predicate.go +++ b/internal/controller/predicate/predicate.go @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 +// Package predicate contains helper functions used for building predicates for +// event filtering for etcd-druid controllers. package predicate import ( From e66045fc49855b8dd5f367214fbf453bcc9973d1 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 17 May 2024 17:21:31 +0530 Subject: [PATCH 187/235] Fix and enhance sentinel webhook unit tests --- internal/webhook/sentinel/handler.go | 5 +- internal/webhook/sentinel/handler_test.go | 104 ++++++++++++++++++++-- 2 files changed, 99 insertions(+), 10 deletions(-) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 2f5bbef00..64449306a 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -79,14 +79,15 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R // More information can be found at https://github.com/kubernetes/kubernetes/issues/113594#issuecomment-1332573990 requestResourceGK := schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} if requestGK == autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind() && - requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() { + requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() && + req.SubResource == "Scale" { sts, err := h.fetchStatefulSet(ctx, req.Name, req.Namespace) if err != nil { return admission.Errored(http.StatusInternalServerError, err) } managedBy, hasLabel := sts.GetLabels()[druidv1alpha1.LabelManagedByKey] if !hasLabel || managedBy != druidv1alpha1.LabelManagedByValue { - return admission.Allowed(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey)) + return admission.Allowed("statefulset not managed by etcd-druid") } // replicate sts labels to the /scale subresource object, used for subsequent checks obj.SetLabels(sts.GetLabels()) diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index c22f72eb4..ee1dd28fa 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -21,11 +21,14 @@ import ( admissionv1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + coordinationv1 "k8s.io/api/coordination/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -46,8 +49,10 @@ var ( reconcilerServiceAccount = "etcd-druid-sa" exemptServiceAccounts = []string{"exempt-sa-1"} - statefulSetGVK = metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"} - leaseGVK = metav1.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"} + statefulSetGVK = metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"} + statefulSetGVR = metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"} + scaleSubresourceGVK = metav1.GroupVersionKind{Group: "autoscaling", Version: "v1", Kind: "Scale"} + leaseGVK = metav1.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"} ) func TestHandleCreateAndConnect(t *testing.T) { @@ -99,7 +104,81 @@ func TestHandleCreateAndConnect(t *testing.T) { func TestHandleLeaseUpdate(t *testing.T) { g := NewWithT(t) - cl := fake.NewClientBuilder().Build() + + testCases := []struct { + name string + useEtcdServiceAccount bool + expectedAllowed bool + expectedMessage string + expectedCode int32 + }{ + { + name: "request is from Etcd service account", + useEtcdServiceAccount: true, + expectedAllowed: true, + expectedMessage: "lease resource can be freely updated by etcd members", + expectedCode: http.StatusOK, + }, + { + name: "request is not from Etcd service account", + useEtcdServiceAccount: false, + expectedAllowed: false, + expectedMessage: fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", testEtcdName), + expectedCode: http.StatusForbidden, + }, + } + + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, nil, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) + decoder := admission.NewDecoder(cl.Scheme()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), + } + + obj := buildObjRawExtension(g, &coordinationv1.Lease{}, nil, testObjectName, testNamespace, + map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}) + + username := testUserName + if tc.useEtcdServiceAccount { + username = fmt.Sprintf("system:serviceaccount:%s:%s", testNamespace, etcd.GetServiceAccountName()) + } + + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + UserInfo: authenticationv1.UserInfo{Username: username}, + Kind: leaseGVK, + Name: testObjectName, + Namespace: testNamespace, + Object: obj, + OldObject: obj, + }, + }) + + g.Expect(response.Allowed).To(Equal(tc.expectedAllowed)) + g.Expect(response.Result.Message).To(ContainSubstring(tc.expectedMessage)) + g.Expect(response.Result.Code).To(Equal(tc.expectedCode)) + }) + } +} + +func TestHandleStatefulSetScaleSubresourceUpdate(t *testing.T) { + g := NewWithT(t) + + // create sts without managed-by label + sts := testutils.CreateStatefulSet(testEtcdName, testNamespace, uuid.NewUUID(), 1) + delete(sts.Labels, druidv1alpha1.LabelManagedByKey) + + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, nil, nil, nil, nil, []client.Object{sts}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) decoder := admission.NewDecoder(cl.Scheme()) handler := &Handler{ @@ -111,15 +190,24 @@ func TestHandleLeaseUpdate(t *testing.T) { logger: logr.Discard(), } - resp := handler.Handle(context.Background(), admission.Request{ + obj := buildObjRawExtension(g, &autoscalingv1.Scale{}, nil, testObjectName, testNamespace, nil) + + response := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Update, - Kind: leaseGVK, + Operation: admissionv1.Update, + Kind: scaleSubresourceGVK, + Resource: statefulSetGVR, + SubResource: "Scale", + Name: testObjectName, + Namespace: testNamespace, + Object: obj, + OldObject: obj, }, }) - g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(resp.Result.Message).To(Equal("lease resource can be freely updated")) + g.Expect(response.Allowed).To(BeTrue()) + g.Expect(response.Result.Message).To(Equal("statefulset not managed by etcd-druid")) + g.Expect(response.Result.Code).To(Equal(int32(http.StatusOK))) } func TestUnexpectedResourceType(t *testing.T) { From 0440243706b33386ecf0a01acb867337dacff9c5 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 21 May 2024 12:03:33 +0530 Subject: [PATCH 188/235] package renaming, healthz and readyz endpoint addition, partialObjMetadata usage and other smaller changes --- Makefile | 2 +- internal/common/constants.go | 2 +- .../clientservice/clientservice.go | 13 +-- .../clientservice/clientservice_test.go | 2 +- .../configmap/configmap.go | 13 +-- .../configmap/configmap_test.go | 2 +- .../configmap/etcdconfig.go | 0 .../memberlease/memberlease.go | 15 +-- .../memberlease/memberlease_test.go | 6 +- .../peerservice/peerservice.go | 13 +-- .../peerservice/peerservice_test.go | 2 +- .../poddisruptionbudget.go | 13 +-- .../poddisruptionbudget_test.go | 2 +- internal/{operator => component}/registry.go | 28 +++--- internal/{operator => component}/role/role.go | 13 +-- .../{operator => component}/role/role_test.go | 2 +- .../rolebinding/rolebinding.go | 13 +-- .../rolebinding/rolebinding_test.go | 2 +- .../serviceaccount/serviceaccount.go | 13 +-- .../serviceaccount/serviceaccount_test.go | 2 +- .../snapshotlease/snapshotlease.go | 17 ++-- .../snapshotlease/snapshotlease_test.go | 2 +- .../statefulset/builder.go | 2 +- .../statefulset/constants.go | 0 .../statefulset/statefulset.go | 26 ++--- .../statefulset/statefulset_test.go | 2 +- .../statefulset/stsmatcher.go | 0 internal/{operator => }/component/types.go | 0 internal/controller/etcd/reconcile_delete.go | 6 +- internal/controller/etcd/reconcile_spec.go | 29 +++--- internal/controller/etcd/reconcile_status.go | 10 +- internal/controller/etcd/reconciler.go | 57 ++++++----- internal/controller/utils/etcdstatus.go | 20 ++-- internal/manager/config.go | 74 ++++++++------ internal/manager/manager.go | 36 +++++-- internal/utils/concurrent.go | 4 +- main.go | 5 +- test/it/controller/etcd/assertions.go | 97 +++++++++---------- test/it/controller/etcd/reconciler_test.go | 46 ++++----- 39 files changed, 316 insertions(+), 275 deletions(-) rename internal/{operator => component}/clientservice/clientservice.go (93%) rename internal/{operator => component}/clientservice/clientservice_test.go (99%) rename internal/{operator => component}/configmap/configmap.go (92%) rename internal/{operator => component}/configmap/configmap_test.go (99%) rename internal/{operator => component}/configmap/etcdconfig.go (100%) rename internal/{operator => component}/memberlease/memberlease.go (94%) rename internal/{operator => component}/memberlease/memberlease_test.go (98%) rename internal/{operator => component}/peerservice/peerservice.go (92%) rename internal/{operator => component}/peerservice/peerservice_test.go (99%) rename internal/{operator => component}/poddistruptionbudget/poddisruptionbudget.go (91%) rename internal/{operator => component}/poddistruptionbudget/poddisruptionbudget_test.go (99%) rename internal/{operator => component}/registry.go (72%) rename internal/{operator => component}/role/role.go (92%) rename internal/{operator => component}/role/role_test.go (99%) rename internal/{operator => component}/rolebinding/rolebinding.go (91%) rename internal/{operator => component}/rolebinding/rolebinding_test.go (99%) rename internal/{operator => component}/serviceaccount/serviceaccount.go (91%) rename internal/{operator => component}/serviceaccount/serviceaccount_test.go (99%) rename internal/{operator => component}/snapshotlease/snapshotlease.go (91%) rename internal/{operator => component}/snapshotlease/snapshotlease_test.go (99%) rename internal/{operator => component}/statefulset/builder.go (99%) rename internal/{operator => component}/statefulset/constants.go (100%) rename internal/{operator => component}/statefulset/statefulset.go (93%) rename internal/{operator => component}/statefulset/statefulset_test.go (98%) rename internal/{operator => component}/statefulset/stsmatcher.go (100%) rename internal/{operator => }/component/types.go (100%) diff --git a/Makefile b/Makefile index 35c627996..dfd312291 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ test: $(GINKGO) $(GOTESTFMT) ./internal/mapper/... \ ./internal/metrics/... # run the golang native unit tests. - @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./internal/controller/etcd/... ./internal/operator/... ./internal/utils/... ./internal/webhook/... + @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./internal/controller/etcd/... ./internal/component/... ./internal/utils/... ./internal/webhook/... .PHONY: test-integration test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) diff --git a/internal/common/constants.go b/internal/common/constants.go index 178d2dc9e..a277f1e8e 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -9,7 +9,7 @@ const ( DefaultImageVectorFilePath = "charts/images.yaml" // FinalizerName is the name of the etcd finalizer. FinalizerName = "druid.gardener.cloud/etcd-druid" - // CheckSumKeyConfigMap is the key that is set by a configmap operator and used by StatefulSet operator to + // CheckSumKeyConfigMap is the key that is set by a configmap component and used by StatefulSet component to // place an annotation on the StatefulSet pods. The value contains the check-sum of the latest configmap that // should be reflected on the pods. CheckSumKeyConfigMap = "checksum/etcd-configmap" diff --git a/internal/operator/clientservice/clientservice.go b/internal/component/clientservice/clientservice.go similarity index 93% rename from internal/operator/clientservice/clientservice.go rename to internal/component/clientservice/clientservice.go index 2763074e7..da9c21e13 100644 --- a/internal/operator/clientservice/clientservice.go +++ b/internal/component/clientservice/clientservice.go @@ -9,8 +9,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -33,7 +33,7 @@ type _resource struct { client client.Client } -// New returns a new client service operator. +// New returns a new client service component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -43,9 +43,10 @@ func New(client client.Client) component.Operator { // GetExistingResourceNames returns the name of the existing client service for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) - svc := &corev1.Service{} svcObjectKey := getObjectKey(etcd) - if err := r.client.Get(ctx, svcObjectKey, svc); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) + if err := r.client.Get(ctx, svcObjectKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -54,8 +55,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting client service: %v for etcd: %v", svcObjectKey, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(svc, etcd) { - resourceNames = append(resourceNames, svc.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/clientservice/clientservice_test.go b/internal/component/clientservice/clientservice_test.go similarity index 99% rename from internal/operator/clientservice/clientservice_test.go rename to internal/component/clientservice/clientservice_test.go index 7dc344fd9..11ab7fba3 100644 --- a/internal/operator/clientservice/clientservice_test.go +++ b/internal/component/clientservice/clientservice_test.go @@ -10,8 +10,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" diff --git a/internal/operator/configmap/configmap.go b/internal/component/configmap/configmap.go similarity index 92% rename from internal/operator/configmap/configmap.go rename to internal/component/configmap/configmap.go index c03b140c0..4df4d3d36 100644 --- a/internal/operator/configmap/configmap.go +++ b/internal/component/configmap/configmap.go @@ -10,8 +10,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" gardenerutils "github.com/gardener/gardener/pkg/utils" @@ -37,7 +37,7 @@ type _resource struct { client client.Client } -// New returns a new configmap operator. +// New returns a new configmap component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -48,8 +48,9 @@ func New(client client.Client) component.Operator { func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objKey := getObjectKey(etcd) - cm := &corev1.ConfigMap{} - if err := r.client.Get(ctx, objKey, cm); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) + if err := r.client.Get(ctx, objKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -58,8 +59,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting ConfigMap: %v for etcd: %v", objKey, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(cm, etcd) { - resourceNames = append(resourceNames, cm.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go similarity index 99% rename from internal/operator/configmap/configmap_test.go rename to internal/component/configmap/configmap_test.go index 66cc92bff..a25a9ff78 100644 --- a/internal/operator/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -12,8 +12,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" diff --git a/internal/operator/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go similarity index 100% rename from internal/operator/configmap/etcdconfig.go rename to internal/component/configmap/etcdconfig.go diff --git a/internal/operator/memberlease/memberlease.go b/internal/component/memberlease/memberlease.go similarity index 94% rename from internal/operator/memberlease/memberlease.go rename to internal/component/memberlease/memberlease.go index b623ea17d..9738c2264 100644 --- a/internal/operator/memberlease/memberlease.go +++ b/internal/component/memberlease/memberlease.go @@ -9,12 +9,11 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" - "github.com/hashicorp/go-multierror" - "github.com/gardener/gardener/pkg/controllerutils" + "github.com/hashicorp/go-multierror" coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,7 +32,7 @@ type _resource struct { client client.Client } -// New returns a new member lease operator. +// New returns a new member lease component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -43,9 +42,11 @@ func New(client client.Client) component.Operator { // GetExistingResourceNames returns the names of the existing member leases for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) - leaseList := &coordinationv1.LeaseList{} + + objMetaList := &metav1.PartialObjectMetadataList{} + objMetaList.SetGroupVersionKind(coordinationv1.SchemeGroupVersion.WithKind("Lease")) if err := r.client.List(ctx, - leaseList, + objMetaList, client.InNamespace(etcd.Namespace), client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd)), ); err != nil { @@ -54,7 +55,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error listing member leases for etcd: %v", etcd.GetNamespaceName())) } - for _, lease := range leaseList.Items { + for _, lease := range objMetaList.Items { if metav1.IsControlledBy(&lease, etcd) { resourceNames = append(resourceNames, lease.Name) } diff --git a/internal/operator/memberlease/memberlease_test.go b/internal/component/memberlease/memberlease_test.go similarity index 98% rename from internal/operator/memberlease/memberlease_test.go rename to internal/component/memberlease/memberlease_test.go index 077304325..37d49c18a 100644 --- a/internal/operator/memberlease/memberlease_test.go +++ b/internal/component/memberlease/memberlease_test.go @@ -12,8 +12,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" @@ -173,7 +173,7 @@ func TestSync(t *testing.T) { } else { updatedEtcd = etcd } - // ***************** Setup operator and test ***************** + // ***************** Setup component operator and test ***************** operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, updatedEtcd) @@ -250,7 +250,7 @@ func TestTriggerDelete(t *testing.T) { existingObjects = append(existingObjects, testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.ComponentNameMemberLease)) } cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd), existingObjects...) - // ***************** Setup operator and test ***************** + // ***************** Setup component operator and test ***************** operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.TriggerDelete(opCtx, etcd) diff --git a/internal/operator/peerservice/peerservice.go b/internal/component/peerservice/peerservice.go similarity index 92% rename from internal/operator/peerservice/peerservice.go rename to internal/component/peerservice/peerservice.go index 6f6984e59..293425296 100644 --- a/internal/operator/peerservice/peerservice.go +++ b/internal/component/peerservice/peerservice.go @@ -9,8 +9,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -33,7 +33,7 @@ type _resource struct { client client.Client } -// New returns a new peer service operator. +// New returns a new peer service component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -44,8 +44,9 @@ func New(client client.Client) component.Operator { func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) svcObjectKey := getObjectKey(etcd) - svc := &corev1.Service{} - if err := r.client.Get(ctx, svcObjectKey, svc); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) + if err := r.client.Get(ctx, svcObjectKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -54,8 +55,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting peer service: %s for etcd: %v", svcObjectKey.Name, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(svc, etcd) { - resourceNames = append(resourceNames, svc.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/peerservice/peerservice_test.go b/internal/component/peerservice/peerservice_test.go similarity index 99% rename from internal/operator/peerservice/peerservice_test.go rename to internal/component/peerservice/peerservice_test.go index 7c84ef2b1..571891a0c 100644 --- a/internal/operator/peerservice/peerservice_test.go +++ b/internal/component/peerservice/peerservice_test.go @@ -11,8 +11,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget.go b/internal/component/poddistruptionbudget/poddisruptionbudget.go similarity index 91% rename from internal/operator/poddistruptionbudget/poddisruptionbudget.go rename to internal/component/poddistruptionbudget/poddisruptionbudget.go index 403a18e33..8f09f7aa5 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/component/poddistruptionbudget/poddisruptionbudget.go @@ -9,8 +9,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" policyv1 "k8s.io/api/policy/v1" @@ -33,7 +33,7 @@ type _resource struct { client client.Client } -// New returns a new pod disruption budget operator. +// New returns a new pod disruption budget component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -44,8 +44,9 @@ func New(client client.Client) component.Operator { func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) - pdb := &policyv1.PodDisruptionBudget{} - if err := r.client.Get(ctx, objectKey, pdb); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget")) + if err := r.client.Get(ctx, objectKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -54,8 +55,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(pdb, etcd) { - resourceNames = append(resourceNames, pdb.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go b/internal/component/poddistruptionbudget/poddisruptionbudget_test.go similarity index 99% rename from internal/operator/poddistruptionbudget/poddisruptionbudget_test.go rename to internal/component/poddistruptionbudget/poddisruptionbudget_test.go index 8d8b67e26..2abd1c425 100644 --- a/internal/operator/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/component/poddistruptionbudget/poddisruptionbudget_test.go @@ -9,8 +9,8 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" diff --git a/internal/operator/registry.go b/internal/component/registry.go similarity index 72% rename from internal/operator/registry.go rename to internal/component/registry.go index 434bf2b5e..1f756742a 100644 --- a/internal/operator/registry.go +++ b/internal/component/registry.go @@ -2,21 +2,17 @@ // // SPDX-License-Identifier: Apache-2.0 -package operator - -import ( - "github.com/gardener/etcd-druid/internal/operator/component" -) +package component // Registry is a facade which gives access to all component operators. type Registry interface { - // Register provides consumers to register an operator against the kind of component it operates on. - Register(kind Kind, operator component.Operator) + // Register provides consumers to register a component operator against the kind of component it operates on. + Register(kind Kind, operator Operator) // AllOperators gives a map, where the key is the Kind of component that an operator manages and the value is an Operator itself. - AllOperators() map[Kind]component.Operator - // GetOperator gets an operator that operates on the kind. - // Returns the operator if an operator is found, else nil will be returned. - GetOperator(kind Kind) component.Operator + AllOperators() map[Kind]Operator + // GetOperator gets a component operator that operates on the kind. + // Returns the component operator if found, else nil will be returned. + GetOperator(kind Kind) Operator } // Kind represents the kind of component that an operator manages. @@ -46,25 +42,25 @@ const ( ) type registry struct { - operators map[Kind]component.Operator + operators map[Kind]Operator } // NewRegistry creates a new instance of a Registry. func NewRegistry() Registry { - operators := make(map[Kind]component.Operator) + operators := make(map[Kind]Operator) return registry{ operators: operators, } } -func (r registry) Register(kind Kind, operator component.Operator) { +func (r registry) Register(kind Kind, operator Operator) { r.operators[kind] = operator } -func (r registry) GetOperator(kind Kind) component.Operator { +func (r registry) GetOperator(kind Kind) Operator { return r.operators[kind] } -func (r registry) AllOperators() map[Kind]component.Operator { +func (r registry) AllOperators() map[Kind]Operator { return r.operators } diff --git a/internal/operator/role/role.go b/internal/component/role/role.go similarity index 92% rename from internal/operator/role/role.go rename to internal/component/role/role.go index 1271ffd1c..216d601ab 100644 --- a/internal/operator/role/role.go +++ b/internal/component/role/role.go @@ -10,8 +10,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" @@ -33,7 +33,7 @@ type _resource struct { client client.Client } -// New returns a new role operator. +// New returns a new role component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -44,8 +44,9 @@ func New(client client.Client) component.Operator { func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) - role := &rbacv1.Role{} - if err := r.client.Get(ctx, objectKey, role); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := r.client.Get(ctx, objectKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -54,8 +55,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting role: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(role, etcd) { - resourceNames = append(resourceNames, role.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/role/role_test.go b/internal/component/role/role_test.go similarity index 99% rename from internal/operator/role/role_test.go rename to internal/component/role/role_test.go index 1a9ee1b2c..694d6940e 100644 --- a/internal/operator/role/role_test.go +++ b/internal/component/role/role_test.go @@ -9,8 +9,8 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" diff --git a/internal/operator/rolebinding/rolebinding.go b/internal/component/rolebinding/rolebinding.go similarity index 91% rename from internal/operator/rolebinding/rolebinding.go rename to internal/component/rolebinding/rolebinding.go index 947b7c704..472e5e043 100644 --- a/internal/operator/rolebinding/rolebinding.go +++ b/internal/component/rolebinding/rolebinding.go @@ -10,8 +10,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" rbacv1 "k8s.io/api/rbac/v1" @@ -33,7 +33,7 @@ type _resource struct { client client.Client } -// New returns a new role binding operator. +// New returns a new role binding component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -44,8 +44,9 @@ func New(client client.Client) component.Operator { func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) - rb := &rbacv1.RoleBinding{} - if err := r.client.Get(ctx, objectKey, rb); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := r.client.Get(ctx, objectKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -54,8 +55,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(rb, etcd) { - resourceNames = append(resourceNames, rb.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/rolebinding/rolebinding_test.go b/internal/component/rolebinding/rolebinding_test.go similarity index 99% rename from internal/operator/rolebinding/rolebinding_test.go rename to internal/component/rolebinding/rolebinding_test.go index 330c84837..13e8fdd0c 100644 --- a/internal/operator/rolebinding/rolebinding_test.go +++ b/internal/component/rolebinding/rolebinding_test.go @@ -9,8 +9,8 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" diff --git a/internal/operator/serviceaccount/serviceaccount.go b/internal/component/serviceaccount/serviceaccount.go similarity index 91% rename from internal/operator/serviceaccount/serviceaccount.go rename to internal/component/serviceaccount/serviceaccount.go index c11aef500..638805541 100644 --- a/internal/operator/serviceaccount/serviceaccount.go +++ b/internal/component/serviceaccount/serviceaccount.go @@ -9,8 +9,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" @@ -34,7 +34,7 @@ type _resource struct { disableAutoMount bool } -// New returns a new service account operator. +// New returns a new service account component operator. func New(client client.Client, disableAutomount bool) component.Operator { return &_resource{ client: client, @@ -45,9 +45,10 @@ func New(client client.Client, disableAutomount bool) component.Operator { // GetExistingResourceNames returns the name of the existing service account for the given Etcd. func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) - sa := &corev1.ServiceAccount{} objectKey := getObjectKey(etcd) - if err := r.client.Get(ctx, objectKey, sa); err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + if err := r.client.Get(ctx, objectKey, objMeta); err != nil { if errors.IsNotFound(err) { return resourceNames, nil } @@ -56,8 +57,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd "GetExistingResourceNames", fmt.Sprintf("Error getting service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - if metav1.IsControlledBy(sa, etcd) { - resourceNames = append(resourceNames, sa.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } diff --git a/internal/operator/serviceaccount/serviceaccount_test.go b/internal/component/serviceaccount/serviceaccount_test.go similarity index 99% rename from internal/operator/serviceaccount/serviceaccount_test.go rename to internal/component/serviceaccount/serviceaccount_test.go index 77ab0a472..f56896f53 100644 --- a/internal/operator/serviceaccount/serviceaccount_test.go +++ b/internal/component/serviceaccount/serviceaccount_test.go @@ -9,8 +9,8 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" "github.com/google/uuid" diff --git a/internal/operator/snapshotlease/snapshotlease.go b/internal/component/snapshotlease/snapshotlease.go similarity index 91% rename from internal/operator/snapshotlease/snapshotlease.go rename to internal/component/snapshotlease/snapshotlease.go index ef5a27eb3..4dd828c91 100644 --- a/internal/operator/snapshotlease/snapshotlease.go +++ b/internal/component/snapshotlease/snapshotlease.go @@ -11,8 +11,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" coordinationv1 "k8s.io/api/coordination/v1" @@ -34,7 +34,7 @@ type _resource struct { client client.Client } -// New returns a new snapshot lease operator. +// New returns a new snapshot lease component operator. func New(client client.Client) component.Operator { return &_resource{ client: client, @@ -49,7 +49,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // we will add the labels. // TODO: Once all snapshot leases have a purpose label on them, then we can use List instead of individual Get calls. deltaSnapshotObjectKey := client.ObjectKey{Name: etcd.GetDeltaSnapshotLeaseName(), Namespace: etcd.Namespace} - deltaSnapshotLease, err := r.getLease(ctx, deltaSnapshotObjectKey) + deltaSnapshotLease, err := r.getLeasePartialObjectMetadata(ctx, deltaSnapshotObjectKey) if err != nil { return resourceNames, &druiderr.DruidError{ Code: ErrGetSnapshotLease, @@ -62,7 +62,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd resourceNames = append(resourceNames, deltaSnapshotLease.Name) } fullSnapshotObjectKey := client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace} - fullSnapshotLease, err := r.getLease(ctx, fullSnapshotObjectKey) + fullSnapshotLease, err := r.getLeasePartialObjectMetadata(ctx, fullSnapshotObjectKey) if err != nil { return resourceNames, &druiderr.DruidError{ Code: ErrGetSnapshotLease, @@ -130,15 +130,16 @@ func (r _resource) deleteAllSnapshotLeases(ctx component.OperatorContext, etcd * return nil } -func (r _resource) getLease(ctx context.Context, objectKey client.ObjectKey) (*coordinationv1.Lease, error) { - lease := &coordinationv1.Lease{} - if err := r.client.Get(ctx, objectKey, lease); err != nil { +func (r _resource) getLeasePartialObjectMetadata(ctx context.Context, objectKey client.ObjectKey) (*metav1.PartialObjectMetadata, error) { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(coordinationv1.SchemeGroupVersion.WithKind("Lease")) + if err := r.client.Get(ctx, objectKey, objMeta); err != nil { if apierrors.IsNotFound(err) { return nil, nil } return nil, err } - return lease, nil + return objMeta, nil } func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, leaseObjectKey client.ObjectKey) error { diff --git a/internal/operator/snapshotlease/snapshotlease_test.go b/internal/component/snapshotlease/snapshotlease_test.go similarity index 99% rename from internal/operator/snapshotlease/snapshotlease_test.go rename to internal/component/snapshotlease/snapshotlease_test.go index abedca6f6..58ab87458 100644 --- a/internal/operator/snapshotlease/snapshotlease_test.go +++ b/internal/component/snapshotlease/snapshotlease_test.go @@ -11,8 +11,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" diff --git a/internal/operator/statefulset/builder.go b/internal/component/statefulset/builder.go similarity index 99% rename from internal/operator/statefulset/builder.go rename to internal/component/statefulset/builder.go index 05445430d..a239598e8 100644 --- a/internal/operator/statefulset/builder.go +++ b/internal/component/statefulset/builder.go @@ -11,7 +11,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/operator/component" + "github.com/gardener/etcd-druid/internal/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" diff --git a/internal/operator/statefulset/constants.go b/internal/component/statefulset/constants.go similarity index 100% rename from internal/operator/statefulset/constants.go rename to internal/component/statefulset/constants.go diff --git a/internal/operator/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go similarity index 93% rename from internal/operator/statefulset/statefulset.go rename to internal/component/statefulset/statefulset.go index cf01b7ee1..c207cb3b0 100644 --- a/internal/operator/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -9,15 +9,14 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,7 +37,7 @@ type _resource struct { useEtcdWrapper bool } -// New returns a new statefulset resource. +// New returns a new statefulset component operator. func New(client client.Client, imageVector imagevector.ImageVector, featureGates map[featuregate.Feature]bool) component.Operator { return &_resource{ client: client, @@ -51,18 +50,19 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { resourceNames := make([]string, 0, 1) objectKey := getObjectKey(etcd) - sts, err := r.getExistingStatefulSet(ctx, etcd) - if err != nil { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) + if err := r.client.Get(ctx, objectKey, objMeta); err != nil { + if apierrors.IsNotFound(err) { + return resourceNames, nil + } return nil, druiderr.WrapError(err, ErrGetStatefulSet, "GetExistingResourceNames", fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) } - if sts == nil { - return resourceNames, nil - } - if metav1.IsControlledBy(sts, etcd) { - resourceNames = append(resourceNames, sts.Name) + if metav1.IsControlledBy(objMeta, etcd) { + resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil } @@ -101,7 +101,7 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp objectKey := getObjectKey(etcd) ctx.Logger.Info("Triggering deletion of StatefulSet", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyStatefulSet(etcd)); err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { ctx.Logger.Info("No StatefulSet found, Deletion is a No-Op", "objectKey", objectKey.Name) return nil } @@ -119,7 +119,7 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { sts := emptyStatefulSet(etcd) if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return nil, nil } return nil, err diff --git a/internal/operator/statefulset/statefulset_test.go b/internal/component/statefulset/statefulset_test.go similarity index 98% rename from internal/operator/statefulset/statefulset_test.go rename to internal/component/statefulset/statefulset_test.go index 80aaf533f..cdc91365b 100644 --- a/internal/operator/statefulset/statefulset_test.go +++ b/internal/component/statefulset/statefulset_test.go @@ -10,9 +10,9 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/go-logr/logr" diff --git a/internal/operator/statefulset/stsmatcher.go b/internal/component/statefulset/stsmatcher.go similarity index 100% rename from internal/operator/statefulset/stsmatcher.go rename to internal/component/statefulset/stsmatcher.go diff --git a/internal/operator/component/types.go b/internal/component/types.go similarity index 100% rename from internal/operator/component/types.go rename to internal/component/types.go diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 1b4462911..8df09325e 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -10,8 +10,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" @@ -45,7 +45,7 @@ func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjK // TODO: once we move to go 1.22 (https://go.dev/blog/loopvar-preview) operator := operator deleteTasks = append(deleteTasks, utils.OperatorTask{ - Name: fmt.Sprintf("triggerDeletionFlow-%s-operator", kind), + Name: fmt.Sprintf("triggerDeletionFlow-%s-component", kind), Fn: func(ctx component.OperatorContext) error { return operator.TriggerDelete(ctx, etcd) }, @@ -109,7 +109,7 @@ func (r *Reconciler) recordIncompleteDeletionOperation(ctx component.OperatorCon if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if err := r.lastOpErrRecorder.RecordError(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeDelete, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { + if err := r.lastOpErrRecorder.RecordErrors(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeDelete, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { logger.Error(err, "failed to record last operation and last errors for etcd deletion") return ctrlutils.ReconcileWithError(err) } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index db3ba0faa..07292bcb1 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -6,9 +6,8 @@ package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" - "github.com/gardener/etcd-druid/internal/operator" - "github.com/gardener/etcd-druid/internal/operator/component" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -96,7 +95,7 @@ func (r *Reconciler) recordReconcileSuccessOperation(ctx component.OperatorConte } func (r *Reconciler) recordIncompleteReconcileOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { - if err := r.lastOpErrRecorder.RecordError(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { + if err := r.lastOpErrRecorder.RecordErrors(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { ctx.Logger.Error(err, "failed to record last operation and last errors for etcd reconciliation") return ctrlutils.ReconcileWithError(err) } @@ -154,18 +153,18 @@ func (r *Reconciler) recordEtcdSpecReconcileSuspension(etcd *druidv1alpha1.Etcd, ) } -func (r *Reconciler) getOrderedOperatorsForSync() []operator.Kind { - return []operator.Kind{ - operator.MemberLeaseKind, - operator.SnapshotLeaseKind, - operator.ClientServiceKind, - operator.PeerServiceKind, - operator.ConfigMapKind, - operator.PodDisruptionBudgetKind, - operator.ServiceAccountKind, - operator.RoleKind, - operator.RoleBindingKind, - operator.StatefulSetKind, +func (r *Reconciler) getOrderedOperatorsForSync() []component.Kind { + return []component.Kind{ + component.MemberLeaseKind, + component.SnapshotLeaseKind, + component.ClientServiceKind, + component.PeerServiceKind, + component.ConfigMapKind, + component.PodDisruptionBudgetKind, + component.ServiceAccountKind, + component.RoleKind, + component.RoleBindingKind, + component.StatefulSetKind, } } diff --git a/internal/controller/etcd/reconcile_status.go b/internal/controller/etcd/reconcile_status.go index 08032d804..27d245145 100644 --- a/internal/controller/etcd/reconcile_status.go +++ b/internal/controller/etcd/reconcile_status.go @@ -6,17 +6,17 @@ package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/health/status" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/go-logr/logr" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) -// mutateEtcdFn is a function which mutates the status of the passed etcd object -type mutateEtcdFn func(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, logger logr.Logger) ctrlutils.ReconcileStepResult +// mutateEtcdStatusFn is a function which mutates the status of the passed etcd object +type mutateEtcdStatusFn func(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, logger logr.Logger) ctrlutils.ReconcileStepResult func (r *Reconciler) reconcileStatus(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} @@ -25,11 +25,11 @@ func (r *Reconciler) reconcileStatus(ctx component.OperatorContext, etcdObjectKe } sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) originalEtcd := etcd.DeepCopy() - mutateETCDStepFns := []mutateEtcdFn{ + mutateETCDStatusStepFns := []mutateEtcdStatusFn{ r.mutateETCDStatusWithMemberStatusAndConditions, r.inspectStatefulSetAndMutateETCDStatus, } - for _, fn := range mutateETCDStepFns { + for _, fn := range mutateETCDStatusStepFns { if stepResult := fn(ctx, etcd, sLog); ctrlutils.ShortCircuitReconcileFlow(stepResult) { return stepResult } diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 2e5d7ebec..0d0f7e5da 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -8,20 +8,19 @@ import ( "context" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" + "github.com/gardener/etcd-druid/internal/component/clientservice" + "github.com/gardener/etcd-druid/internal/component/configmap" + "github.com/gardener/etcd-druid/internal/component/memberlease" + "github.com/gardener/etcd-druid/internal/component/peerservice" + "github.com/gardener/etcd-druid/internal/component/poddistruptionbudget" + "github.com/gardener/etcd-druid/internal/component/role" + "github.com/gardener/etcd-druid/internal/component/rolebinding" + "github.com/gardener/etcd-druid/internal/component/serviceaccount" + "github.com/gardener/etcd-druid/internal/component/snapshotlease" + "github.com/gardener/etcd-druid/internal/component/statefulset" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/images" - "github.com/gardener/etcd-druid/internal/operator" - "github.com/gardener/etcd-druid/internal/operator/clientservice" - "github.com/gardener/etcd-druid/internal/operator/component" - "github.com/gardener/etcd-druid/internal/operator/configmap" - "github.com/gardener/etcd-druid/internal/operator/memberlease" - "github.com/gardener/etcd-druid/internal/operator/peerservice" - "github.com/gardener/etcd-druid/internal/operator/poddistruptionbudget" - "github.com/gardener/etcd-druid/internal/operator/role" - "github.com/gardener/etcd-druid/internal/operator/rolebinding" - "github.com/gardener/etcd-druid/internal/operator/serviceaccount" - "github.com/gardener/etcd-druid/internal/operator/snapshotlease" - "github.com/gardener/etcd-druid/internal/operator/statefulset" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" "github.com/google/uuid" @@ -39,8 +38,8 @@ type Reconciler struct { config *Config recorder record.EventRecorder imageVector imagevector.ImageVector - operatorRegistry operator.Registry - lastOpErrRecorder ctrlutils.LastOperationErrorRecorder + operatorRegistry component.Registry + lastOpErrRecorder ctrlutils.LastOperationAndLastErrorsRecorder logger logr.Logger } @@ -57,7 +56,7 @@ func NewReconciler(mgr manager.Manager, config *Config) (*Reconciler, error) { func NewReconcilerWithImageVector(mgr manager.Manager, config *Config, iv imagevector.ImageVector) (*Reconciler, error) { logger := log.Log.WithName(controllerName) operatorReg := createAndInitializeOperatorRegistry(mgr.GetClient(), config, iv) - lastOpErrRecorder := ctrlutils.NewLastOperationErrorRecorder(mgr.GetClient(), logger) + lastOpErrRecorder := ctrlutils.NewLastOperationAndLastErrorsRecorder(mgr.GetClient(), logger) return &Reconciler{ client: mgr.GetClient(), config: config, @@ -114,23 +113,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult() } -// GetOperatorRegistry returns the operator registry. -func (r *Reconciler) GetOperatorRegistry() operator.Registry { +// GetOperatorRegistry returns the component registry. +func (r *Reconciler) GetOperatorRegistry() component.Registry { return r.operatorRegistry } -func createAndInitializeOperatorRegistry(client client.Client, config *Config, imageVector imagevector.ImageVector) operator.Registry { - reg := operator.NewRegistry() - reg.Register(operator.ConfigMapKind, configmap.New(client)) - reg.Register(operator.ServiceAccountKind, serviceaccount.New(client, config.DisableEtcdServiceAccountAutomount)) - reg.Register(operator.MemberLeaseKind, memberlease.New(client)) - reg.Register(operator.SnapshotLeaseKind, snapshotlease.New(client)) - reg.Register(operator.ClientServiceKind, clientservice.New(client)) - reg.Register(operator.PeerServiceKind, peerservice.New(client)) - reg.Register(operator.PodDisruptionBudgetKind, poddistruptionbudget.New(client)) - reg.Register(operator.RoleKind, role.New(client)) - reg.Register(operator.RoleBindingKind, rolebinding.New(client)) - reg.Register(operator.StatefulSetKind, statefulset.New(client, imageVector, config.FeatureGates)) +func createAndInitializeOperatorRegistry(client client.Client, config *Config, imageVector imagevector.ImageVector) component.Registry { + reg := component.NewRegistry() + reg.Register(component.ConfigMapKind, configmap.New(client)) + reg.Register(component.ServiceAccountKind, serviceaccount.New(client, config.DisableEtcdServiceAccountAutomount)) + reg.Register(component.MemberLeaseKind, memberlease.New(client)) + reg.Register(component.SnapshotLeaseKind, snapshotlease.New(client)) + reg.Register(component.ClientServiceKind, clientservice.New(client)) + reg.Register(component.PeerServiceKind, peerservice.New(client)) + reg.Register(component.PodDisruptionBudgetKind, poddistruptionbudget.New(client)) + reg.Register(component.RoleKind, role.New(client)) + reg.Register(component.RoleBindingKind, rolebinding.New(client)) + reg.Register(component.StatefulSetKind, statefulset.New(client, imageVector, config.FeatureGates)) return reg } diff --git a/internal/controller/utils/etcdstatus.go b/internal/controller/utils/etcdstatus.go index 75a5f7819..a8f439dc5 100644 --- a/internal/controller/utils/etcdstatus.go +++ b/internal/controller/utils/etcdstatus.go @@ -8,25 +8,25 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -// LastOperationErrorRecorder records etcd.Status.LastOperation and etcd.Status.LastErrors -type LastOperationErrorRecorder interface { - // RecordStart records the start of an operation in the Etcd status +// LastOperationAndLastErrorsRecorder records etcd.Status.LastOperation and etcd.Status.LastErrors. +type LastOperationAndLastErrorsRecorder interface { + // RecordStart records the start of an operation in the Etcd status. RecordStart(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error - // RecordSuccess records the success of an operation in the Etcd status + // RecordSuccess records the success of an operation in the Etcd status. RecordSuccess(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType) error - // RecordError records an error encountered in the last operation in the Etcd status - RecordError(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error + // RecordErrors records errors encountered in the last operation in the Etcd status. + RecordErrors(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error } -// NewLastOperationErrorRecorder returns a new LastOperationErrorRecorder -func NewLastOperationErrorRecorder(client client.Client, logger logr.Logger) LastOperationErrorRecorder { +// NewLastOperationAndLastErrorsRecorder returns a new LastOperationAndLastErrorsRecorder. +func NewLastOperationAndLastErrorsRecorder(client client.Client, logger logr.Logger) LastOperationAndLastErrorsRecorder { return &lastOpErrRecorder{ client: client, logger: logger, @@ -69,7 +69,7 @@ func (l *lastOpErrRecorder) RecordSuccess(ctx component.OperatorContext, etcdObj return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateSucceeded, description) } -func (l *lastOpErrRecorder) RecordError(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { +func (l *lastOpErrRecorder) RecordErrors(ctx component.OperatorContext, etcdObjectKey client.ObjectKey, operationType druidv1alpha1.LastOperationType, description string, errs ...error) error { description += " Operation will be retried." lastErrors := druiderr.MapToLastErrors(errs) return l.recordLastOperationAndErrors(ctx, etcdObjectKey, operationType, druidv1alpha1.LastOperationStateError, description, lastErrors...) diff --git a/internal/manager/config.go b/internal/manager/config.go index fe2663d02..11281c2de 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -99,16 +99,34 @@ type Config struct { DisableLeaseCache bool // FeatureGates contains the feature gates to be used by etcd-druid. FeatureGates featuregate.MutableFeatureGate - // EtcdControllerConfig is the configuration required for etcd controller. - EtcdControllerConfig *etcd.Config - // CompactionControllerConfig is the configuration required for compaction controller. - CompactionControllerConfig *compaction.Config - // EtcdCopyBackupsTaskControllerConfig is the configuration required for etcd-copy-backup-tasks controller. - EtcdCopyBackupsTaskControllerConfig *etcdcopybackupstask.Config - // SecretControllerConfig is the configuration required for secret controller. - SecretControllerConfig *secret.Config - // SentinelWebhookConfig is the configuration required for sentinel webhook. - SentinelWebhookConfig *sentinel.Config + // Controllers defines the configuration for etcd-druid controllers. + Controllers Controllers + // Webhooks defines the configuration for etcd-druid webhooks. + Webhooks Webhooks +} + +// Controllers defines the configuration for etcd druid controllers. +type Controllers struct { + // Etcd is the configuration required for etcd controller. + Etcd *etcd.Config + // Compaction is the configuration required for compaction controller. + Compaction *compaction.Config + // EtcdCopyBackupsTask is the configuration required for etcd-copy-backup-tasks controller. + EtcdCopyBackupsTask *etcdcopybackupstask.Config + // Secret is the configuration required for secret controller. + Secret *secret.Config +} + +// Webhooks defines the configuration for etcd-druid webhooks. +type Webhooks struct { + // Sentinel is the configuration required for sentinel webhook. + Sentinel *sentinel.Config +} + +// AtLeaseOneEnabled returns true if at least one webhook is enabled. +// NOTE for contributors: For every new webhook, add a disjunction condition with the webhook's AtLeaseOneEnabled field. +func (w Webhooks) AtLeaseOneEnabled() bool { + return w.Sentinel.Enabled } // InitFromFlags initializes the controller manager config from the provided CLI flag set. @@ -143,20 +161,20 @@ func (cfg *Config) InitFromFlags(fs *flag.FlagSet) error { return err } - cfg.EtcdControllerConfig = &etcd.Config{} - etcd.InitFromFlags(fs, cfg.EtcdControllerConfig) + cfg.Controllers.Etcd = &etcd.Config{} + etcd.InitFromFlags(fs, cfg.Controllers.Etcd) - cfg.CompactionControllerConfig = &compaction.Config{} - compaction.InitFromFlags(fs, cfg.CompactionControllerConfig) + cfg.Controllers.Compaction = &compaction.Config{} + compaction.InitFromFlags(fs, cfg.Controllers.Compaction) - cfg.EtcdCopyBackupsTaskControllerConfig = &etcdcopybackupstask.Config{} - etcdcopybackupstask.InitFromFlags(fs, cfg.EtcdCopyBackupsTaskControllerConfig) + cfg.Controllers.EtcdCopyBackupsTask = &etcdcopybackupstask.Config{} + etcdcopybackupstask.InitFromFlags(fs, cfg.Controllers.EtcdCopyBackupsTask) - cfg.SecretControllerConfig = &secret.Config{} - secret.InitFromFlags(fs, cfg.SecretControllerConfig) + cfg.Controllers.Secret = &secret.Config{} + secret.InitFromFlags(fs, cfg.Controllers.Secret) - cfg.SentinelWebhookConfig = &sentinel.Config{} - sentinel.InitFromFlags(fs, cfg.SentinelWebhookConfig) + cfg.Webhooks.Sentinel = &sentinel.Config{} + sentinel.InitFromFlags(fs, cfg.Webhooks.Sentinel) return nil } @@ -179,13 +197,13 @@ func (cfg *Config) populateControllersFeatureGates() { // Feature gates populated only for controllers that use feature gates // Add etcd controller feature gates - cfg.EtcdControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) + cfg.Controllers.Etcd.CaptureFeatureActivations(cfg.FeatureGates) // Add compaction controller feature gates - cfg.CompactionControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) + cfg.Controllers.Compaction.CaptureFeatureActivations(cfg.FeatureGates) // Add etcd-copy-backups-task controller feature gates - cfg.EtcdCopyBackupsTaskControllerConfig.CaptureFeatureActivations(cfg.FeatureGates) + cfg.Controllers.EtcdCopyBackupsTask.CaptureFeatureActivations(cfg.FeatureGates) } // Validate validates the controller manager config. @@ -194,7 +212,7 @@ func (cfg *Config) Validate() error { return err } - if cfg.SentinelWebhookConfig.Enabled { + if cfg.Webhooks.Sentinel.Enabled { if cfg.Server.Webhook.Port == 0 { return fmt.Errorf("webhook port cannot be 0") } @@ -203,19 +221,19 @@ func (cfg *Config) Validate() error { } } - if err := cfg.EtcdControllerConfig.Validate(); err != nil { + if err := cfg.Controllers.Etcd.Validate(); err != nil { return err } - if err := cfg.CompactionControllerConfig.Validate(); err != nil { + if err := cfg.Controllers.Compaction.Validate(); err != nil { return err } - if err := cfg.EtcdCopyBackupsTaskControllerConfig.Validate(); err != nil { + if err := cfg.Controllers.EtcdCopyBackupsTask.Validate(); err != nil { return err } - return cfg.SecretControllerConfig.Validate() + return cfg.Controllers.Secret.Validate() } // getAllowedLeaderElectionResourceLocks returns the allowed resource type to be used for leader election. diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 6ef06a858..23cad89d0 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -7,6 +7,7 @@ package controller import ( "context" "net" + "net/http" "strconv" "strings" "time" @@ -17,6 +18,7 @@ import ( "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" "github.com/gardener/etcd-druid/internal/controller/secret" "github.com/gardener/etcd-druid/internal/webhook/sentinel" + "golang.org/x/exp/slog" coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" corev1 "k8s.io/api/core/v1" @@ -42,12 +44,17 @@ func InitializeManager(config *Config) (ctrl.Manager, error) { if mgr, err = createManager(config); err != nil { return nil, err } + slog.Info("registering controllers and webhooks with manager") + time.Sleep(10 * time.Second) if err = registerControllers(mgr, config); err != nil { return nil, err } if err = registerWebhooks(mgr, config); err != nil { return nil, err } + if err = registerHealthAndReadyEndpoints(mgr, config); err != nil { + return nil, err + } return mgr, nil } @@ -94,7 +101,7 @@ func registerControllers(mgr ctrl.Manager, config *Config) error { var err error // Add etcd reconciler to the manager - etcdReconciler, err := etcd.NewReconciler(mgr, config.EtcdControllerConfig) + etcdReconciler, err := etcd.NewReconciler(mgr, config.Controllers.Etcd) if err != nil { return err } @@ -103,8 +110,8 @@ func registerControllers(mgr ctrl.Manager, config *Config) error { } // Add compaction reconciler to the manager if the CLI flag enable-backup-compaction is true. - if config.CompactionControllerConfig.EnableBackupCompaction { - compactionReconciler, err := compaction.NewReconciler(mgr, config.CompactionControllerConfig) + if config.Controllers.Compaction.EnableBackupCompaction { + compactionReconciler, err := compaction.NewReconciler(mgr, config.Controllers.Compaction) if err != nil { return err } @@ -114,7 +121,7 @@ func registerControllers(mgr ctrl.Manager, config *Config) error { } // Add etcd-copy-backups-task reconciler to the manager - etcdCopyBackupsTaskReconciler, err := etcdcopybackupstask.NewReconciler(mgr, config.EtcdCopyBackupsTaskControllerConfig) + etcdCopyBackupsTaskReconciler, err := etcdcopybackupstask.NewReconciler(mgr, config.Controllers.EtcdCopyBackupsTask) if err != nil { return err } @@ -127,21 +134,36 @@ func registerControllers(mgr ctrl.Manager, config *Config) error { defer cancel() return secret.NewReconciler( mgr, - config.SecretControllerConfig, + config.Controllers.Secret, ).RegisterWithManager(ctx, mgr) } func registerWebhooks(mgr ctrl.Manager, config *Config) error { // Add sentinel webhook to the manager - if config.SentinelWebhookConfig.Enabled { + if config.Webhooks.Sentinel.Enabled { sentinelWebhook, err := sentinel.NewHandler( mgr, - config.SentinelWebhookConfig, + config.Webhooks.Sentinel, ) if err != nil { return err } + slog.Info("Registering Sentinel Webhook with manager") return sentinelWebhook.RegisterWithManager(mgr) } return nil } + +func registerHealthAndReadyEndpoints(mgr ctrl.Manager, config *Config) error { + slog.Info("Registering ping health check endpoint") + if err := mgr.AddHealthzCheck("ping", func(req *http.Request) error { return nil }); err != nil { + return err + } + if config.Webhooks.AtLeaseOneEnabled() { + slog.Info("Registering webhook-server readiness check endpoint") + if err := mgr.AddReadyzCheck("webhook-server", mgr.GetWebhookServer().StartedChecker()); err != nil { + return err + } + } + return nil +} diff --git a/internal/utils/concurrent.go b/internal/utils/concurrent.go index aac51e3aa..9987a1f1d 100644 --- a/internal/utils/concurrent.go +++ b/internal/utils/concurrent.go @@ -9,14 +9,14 @@ import ( "runtime/debug" "sync" - "github.com/gardener/etcd-druid/internal/operator/component" + "github.com/gardener/etcd-druid/internal/component" ) // OperatorTask is a holder for a named function. type OperatorTask struct { // Name is the name of the task Name string - // Fn is the function which accepts an operator context and returns an error if there is one. + // Fn is the function which accepts a component operator context and returns an error if there is one. // Implementations of Fn should handle context cancellation properly. Fn func(ctx component.OperatorContext) error } diff --git a/main.go b/main.go index 5d0f5c04d..9db2ddeb4 100644 --- a/main.go +++ b/main.go @@ -8,13 +8,12 @@ import ( "os" "runtime" + druidmgr "github.com/gardener/etcd-druid/internal/manager" + "github.com/gardener/etcd-druid/internal/version" "github.com/go-logr/logr" "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/log/zap" - druidmgr "github.com/gardener/etcd-druid/internal/manager" - "github.com/gardener/etcd-druid/internal/version" - flag "github.com/spf13/pflag" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index ab652a524..c67b33492 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -15,9 +15,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" "github.com/gardener/etcd-druid/internal/controller/etcd" - "github.com/gardener/etcd-druid/internal/operator" - "github.com/gardener/etcd-druid/internal/operator/component" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/etcd-druid/test/it/setup" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -30,21 +29,21 @@ import ( ) var ( - allComponentKinds = []operator.Kind{ - operator.MemberLeaseKind, - operator.SnapshotLeaseKind, - operator.ClientServiceKind, - operator.PeerServiceKind, - operator.ConfigMapKind, - operator.PodDisruptionBudgetKind, - operator.ServiceAccountKind, - operator.RoleKind, - operator.RoleBindingKind, - operator.StatefulSetKind, + allComponentKinds = []component.Kind{ + component.MemberLeaseKind, + component.SnapshotLeaseKind, + component.ClientServiceKind, + component.PeerServiceKind, + component.ConfigMapKind, + component.PodDisruptionBudgetKind, + component.ServiceAccountKind, + component.RoleKind, + component.RoleBindingKind, + component.StatefulSetKind, } ) -type componentCreatedAssertionFn func(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) +type componentCreatedAssertionFn func(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) // ReconcilerTestEnv represents the test environment for the etcd reconciler. type ReconcilerTestEnv struct { @@ -52,18 +51,18 @@ type ReconcilerTestEnv struct { reconciler *etcd.Reconciler } -func getKindToComponentCreatedAssertionFns() map[operator.Kind]componentCreatedAssertionFn { - return map[operator.Kind]componentCreatedAssertionFn{ - operator.MemberLeaseKind: assertMemberLeasesCreated, - operator.SnapshotLeaseKind: assertSnapshotLeasesCreated, - operator.ClientServiceKind: assertClientServiceCreated, - operator.PeerServiceKind: assertPeerServiceCreated, - operator.ConfigMapKind: assertConfigMapCreated, - operator.PodDisruptionBudgetKind: assertPDBCreated, - operator.ServiceAccountKind: assertServiceAccountCreated, - operator.RoleKind: assertRoleCreated, - operator.RoleBindingKind: assertRoleBindingCreated, - operator.StatefulSetKind: assertStatefulSetCreated, +func getKindToComponentCreatedAssertionFns() map[component.Kind]componentCreatedAssertionFn { + return map[component.Kind]componentCreatedAssertionFn{ + component.MemberLeaseKind: assertMemberLeasesCreated, + component.SnapshotLeaseKind: assertSnapshotLeasesCreated, + component.ClientServiceKind: assertClientServiceCreated, + component.PeerServiceKind: assertPeerServiceCreated, + component.ConfigMapKind: assertConfigMapCreated, + component.PodDisruptionBudgetKind: assertPDBCreated, + component.ServiceAccountKind: assertServiceAccountCreated, + component.RoleKind: assertRoleCreated, + component.RoleBindingKind: assertRoleBindingCreated, + component.StatefulSetKind: assertStatefulSetCreated, } } @@ -81,7 +80,7 @@ func assertAllComponentsExists(ctx context.Context, t *testing.T, rtEnv Reconcil doAssertComponentsExist(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) } -func assertSelectedComponentsExists(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { +func assertSelectedComponentsExists(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []component.Kind, timeout, pollInterval time.Duration) { g := NewWithT(t) assertFns := make([]componentCreatedAssertionFn, 0, len(componentKinds)) for _, kind := range componentKinds { @@ -92,7 +91,7 @@ func assertSelectedComponentsExists(ctx context.Context, t *testing.T, rtEnv Rec doAssertComponentsExist(ctx, t, rtEnv, etcd, assertFns, timeout, pollInterval) } -func assertComponentsDoNotExist(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []operator.Kind, timeout, pollInterval time.Duration) { +func assertComponentsDoNotExist(ctx context.Context, t *testing.T, rtEnv ReconcilerTestEnv, etcd *druidv1alpha1.Etcd, componentKinds []component.Kind, timeout, pollInterval time.Duration) { opRegistry := rtEnv.reconciler.GetOperatorRegistry() opCtx := component.NewOperatorContext(ctx, rtEnv.itTestEnv.GetLogger(), t.Name()) for _, kind := range componentKinds { @@ -115,7 +114,7 @@ func doAssertComponentsExist(ctx context.Context, t *testing.T, rtEnv Reconciler wg.Wait() } -func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, kind operator.Kind, etcd *druidv1alpha1.Etcd, expectedResourceNames []string, timeout, pollInterval time.Duration) { +func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, kind component.Kind, etcd *druidv1alpha1.Etcd, expectedResourceNames []string, timeout, pollInterval time.Duration) { g := NewWithT(t) op := opRegistry.GetOperator(kind) checkFn := func() error { @@ -140,60 +139,60 @@ func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegis g.Eventually(checkFn).Within(timeout).WithPolling(pollInterval).WithContext(ctx).Should(BeNil()) } -func assertMemberLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertMemberLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedMemberLeaseNames := make([]string, 0, etcd.Spec.Replicas) for i := 0; i < int(etcd.Spec.Replicas); i++ { expectedMemberLeaseNames = append(expectedMemberLeaseNames, fmt.Sprintf("%s-%d", etcd.Name, i)) } - assertResourceCreation(ctx, t, opRegistry, operator.MemberLeaseKind, etcd, expectedMemberLeaseNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.MemberLeaseKind, etcd, expectedMemberLeaseNames, timeout, pollInterval) } -func assertSnapshotLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertSnapshotLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedSnapshotLeaseNames := make([]string, 0, 2) if etcd.IsBackupStoreEnabled() { expectedSnapshotLeaseNames = []string{etcd.GetDeltaSnapshotLeaseName(), etcd.GetFullSnapshotLeaseName()} } - assertResourceCreation(ctx, t, opRegistry, operator.SnapshotLeaseKind, etcd, expectedSnapshotLeaseNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.SnapshotLeaseKind, etcd, expectedSnapshotLeaseNames, timeout, pollInterval) } -func assertClientServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertClientServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedClientServiceNames := []string{etcd.GetClientServiceName()} - assertResourceCreation(ctx, t, opRegistry, operator.ClientServiceKind, etcd, expectedClientServiceNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.ClientServiceKind, etcd, expectedClientServiceNames, timeout, pollInterval) } -func assertPeerServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertPeerServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedPeerServiceNames := []string{etcd.GetPeerServiceName()} - assertResourceCreation(ctx, t, opRegistry, operator.PeerServiceKind, etcd, expectedPeerServiceNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.PeerServiceKind, etcd, expectedPeerServiceNames, timeout, pollInterval) } -func assertConfigMapCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertConfigMapCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedConfigMapNames := []string{etcd.GetConfigMapName()} - assertResourceCreation(ctx, t, opRegistry, operator.ConfigMapKind, etcd, expectedConfigMapNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.ConfigMapKind, etcd, expectedConfigMapNames, timeout, pollInterval) } -func assertPDBCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertPDBCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedPDBNames := []string{etcd.Name} - assertResourceCreation(ctx, t, opRegistry, operator.PodDisruptionBudgetKind, etcd, expectedPDBNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.PodDisruptionBudgetKind, etcd, expectedPDBNames, timeout, pollInterval) } -func assertServiceAccountCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertServiceAccountCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedServiceAccountNames := []string{etcd.GetServiceAccountName()} - assertResourceCreation(ctx, t, opRegistry, operator.ServiceAccountKind, etcd, expectedServiceAccountNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.ServiceAccountKind, etcd, expectedServiceAccountNames, timeout, pollInterval) } -func assertRoleCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertRoleCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedRoleNames := []string{etcd.GetRoleName()} - assertResourceCreation(ctx, t, opRegistry, operator.RoleKind, etcd, expectedRoleNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.RoleKind, etcd, expectedRoleNames, timeout, pollInterval) } -func assertRoleBindingCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertRoleBindingCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedRoleBindingNames := []string{etcd.GetRoleBindingName()} - assertResourceCreation(ctx, t, opRegistry, operator.RoleBindingKind, etcd, expectedRoleBindingNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.RoleBindingKind, etcd, expectedRoleBindingNames, timeout, pollInterval) } -func assertStatefulSetCreated(ctx component.OperatorContext, t *testing.T, opRegistry operator.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { +func assertStatefulSetCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedSTSNames := []string{etcd.Name} - assertResourceCreation(ctx, t, opRegistry, operator.StatefulSetKind, etcd, expectedSTSNames, timeout, pollInterval) + assertResourceCreation(ctx, t, opRegistry, component.StatefulSetKind, etcd, expectedSTSNames, timeout, pollInterval) } func assertStatefulSetGeneration(ctx context.Context, t *testing.T, cl client.Client, stsObjectKey client.ObjectKey, expectedGeneration int64, timeout, pollInterval time.Duration) { diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 512084415..d85f59107 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -15,9 +15,9 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/component" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/features" - "github.com/gardener/etcd-druid/internal/operator" "github.com/gardener/etcd-druid/test/it/controller/assets" "github.com/gardener/etcd-druid/test/it/setup" testutils "github.com/gardener/etcd-druid/test/utils" @@ -139,18 +139,18 @@ func testFailureToCreateAllResources(t *testing.T, testNs string, reconcilerTest // create etcdInstance resource g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) // ***************** test etcd spec reconciliation ***************** - componentKindCreated := []operator.Kind{operator.MemberLeaseKind} + componentKindCreated := []component.Kind{component.MemberLeaseKind} assertSelectedComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, componentKindCreated, timeout, pollingInterval) - componentKindNotCreated := []operator.Kind{ - operator.SnapshotLeaseKind, // no backup store has been set - operator.ClientServiceKind, - operator.PeerServiceKind, - operator.ConfigMapKind, - operator.PodDisruptionBudgetKind, - operator.ServiceAccountKind, - operator.RoleKind, - operator.RoleBindingKind, - operator.StatefulSetKind, + componentKindNotCreated := []component.Kind{ + component.SnapshotLeaseKind, // no backup store has been set + component.ClientServiceKind, + component.PeerServiceKind, + component.ConfigMapKind, + component.PodDisruptionBudgetKind, + component.ServiceAccountKind, + component.RoleKind, + component.RoleBindingKind, + component.StatefulSetKind, } assertComponentsDoNotExist(ctx, t, reconcilerTestEnv, etcdInstance, componentKindNotCreated, timeout, pollingInterval) assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), nil, 5*time.Second, 1*time.Second) @@ -342,17 +342,17 @@ func testPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi g.Expect(cl.Delete(ctx, etcdInstance)).To(Succeed()) t.Logf("successfully marked etcd instance for deletion: %s, waiting for resources to be removed...", etcdInstance.Name) // assert removal of all components except client service and snapshot lease. - assertComponentsDoNotExist(ctx, t, reconcilerTestEnv, etcdInstance, []operator.Kind{ - operator.MemberLeaseKind, - operator.PeerServiceKind, - operator.ConfigMapKind, - operator.PodDisruptionBudgetKind, - operator.ServiceAccountKind, - operator.RoleKind, - operator.RoleBindingKind, - operator.StatefulSetKind}, 2*time.Minute, 2*time.Second) - - assertSelectedComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, []operator.Kind{operator.ClientServiceKind, operator.SnapshotLeaseKind}, 2*time.Minute, 2*time.Second) + assertComponentsDoNotExist(ctx, t, reconcilerTestEnv, etcdInstance, []component.Kind{ + component.MemberLeaseKind, + component.PeerServiceKind, + component.ConfigMapKind, + component.PodDisruptionBudgetKind, + component.ServiceAccountKind, + component.RoleKind, + component.RoleBindingKind, + component.StatefulSetKind}, 2*time.Minute, 2*time.Second) + + assertSelectedComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, []component.Kind{component.ClientServiceKind, component.SnapshotLeaseKind}, 2*time.Minute, 2*time.Second) // assert that the last operation and last errors are updated correctly. expectedLastOperation := &druidv1alpha1.LastOperation{ Type: druidv1alpha1.LastOperationTypeDelete, From f44e11b641a674fbf9d9ec6a016c61b9039054b6 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 22 May 2024 09:00:46 +0530 Subject: [PATCH 189/235] removed unused method from api --- api/v1alpha1/types_etcd.go | 7 ------- api/v1alpha1/types_etcd_test.go | 33 --------------------------------- 2 files changed, 40 deletions(-) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 58c653b95..6a44ccdb8 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -590,13 +590,6 @@ func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { return nil } -// IsSpecReconciliationSuspended returns true if the Etcd resource has the annotation set to suspend spec reconciliation, -// else returns false. -func (e *Etcd) IsSpecReconciliationSuspended() bool { - suspendReconcileAnnotKey := e.GetSuspendEtcdSpecReconcileAnnotationKey() - return suspendReconcileAnnotKey != nil -} - // AreManagedResourcesProtected returns false if the Etcd resource has the `druid.gardener.cloud/disable-resource-protection` annotation set, // else returns true. func (e *Etcd) AreManagedResourcesProtected() bool { diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index aa9833891..2ec641ac7 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -177,39 +177,6 @@ var _ = Describe("Etcd", func() { }) }) - Context("IsSpecReconciliationSuspended", func() { - Context("when etcd has only suspend-etcd-spec-reconcile annotation set", func() { - It("should return true", func() { - etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "", - } - Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) - }) - }) - Context("when etcd has only ignore-reconcile annotation set", func() { - It("should return true", func() { - etcd.Annotations = map[string]string{ - IgnoreReconciliationAnnotation: "true", - } - Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) - }) - }) - Context("when etcd has both suspend-etcd-spec-reconcile and ignore-reconcile annotations set", func() { - It("should return true", func() { - etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "", - IgnoreReconciliationAnnotation: "", - } - Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(true)) - }) - }) - Context("when etcd does not have suspend-etcd-spec-reconcile or annotation druid.gardener.cloud/suspend-etcd-spec-reconcile or druid.gardener.cloud/ignore-reconciliation set", func() { - It("should return false", func() { - Expect(etcd.IsSpecReconciliationSuspended()).To(Equal(false)) - }) - }) - }) - Context("AreManagedResourcesProtected", func() { Context("when etcd has annotation druid.gardener.cloud/disable-resource-protection", func() { It("should return false", func() { From 7857c8a011c0ebbed8f58230f177dccbab8a4e5a Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 22 May 2024 13:47:42 +0530 Subject: [PATCH 190/235] Address review comments by @seshachalam-yv; restructure sentinel webhook handler, allow resource deletion by druid and exempt SAs during etcd deletion --- api/v1alpha1/types_etcd.go | 9 ++ api/v1alpha1/types_etcd_test.go | 60 ++++++++- .../templates/validating-webhook-config.yaml | 5 + docs/development/getting-started-locally.md | 5 + go.mod | 1 + internal/common/constants.go | 2 +- internal/component/statefulset/builder.go | 1 - internal/webhook/sentinel/handler.go | 112 +++++++++------- internal/webhook/sentinel/handler_test.go | 121 +++++++++++++++--- test/it/controller/etcd/reconciler_test.go | 6 +- 10 files changed, 246 insertions(+), 76 deletions(-) diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/types_etcd.go index 6a44ccdb8..d6cd9e2e4 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/types_etcd.go @@ -599,6 +599,15 @@ func (e *Etcd) AreManagedResourcesProtected() bool { // IsReconciliationInProgress returns true if the Etcd resource is currently being reconciled, else returns false. func (e *Etcd) IsReconciliationInProgress() bool { return e.Status.LastOperation != nil && + e.Status.LastOperation.Type == LastOperationTypeReconcile && + (e.Status.LastOperation.State == LastOperationStateProcessing || + e.Status.LastOperation.State == LastOperationStateError) +} + +// IsDeletionInProgress returns true if the Etcd resource is currently being reconciled, else returns false. +func (e *Etcd) IsDeletionInProgress() bool { + return e.Status.LastOperation != nil && + e.Status.LastOperation.Type == LastOperationTypeDelete && (e.Status.LastOperation.State == LastOperationStateProcessing || e.Status.LastOperation.State == LastOperationStateError) } diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go index 2ec641ac7..5ca2ed01f 100644 --- a/api/v1alpha1/types_etcd_test.go +++ b/api/v1alpha1/types_etcd_test.go @@ -194,36 +194,90 @@ var _ = Describe("Etcd", func() { }) Context("IsReconciliationInProgress", func() { - Context("when etcd status has lastOperation and its state is Processing", func() { + Context("when etcd status has lastOperation, its type is Reconcile and its state is Processing", func() { It("should return true", func() { etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeReconcile, State: LastOperationStateProcessing, } Expect(etcd.IsReconciliationInProgress()).To(Equal(true)) }) }) - Context("when etcd status has lastOperation and its state is Error", func() { + Context("when etcd status has lastOperation, its type is Reconcile and its state is Error", func() { It("should return true", func() { etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeReconcile, State: LastOperationStateError, } Expect(etcd.IsReconciliationInProgress()).To(Equal(true)) }) }) - Context("when etcd status has lastOperation and its state is neither Processing or Error", func() { + Context("when etcd status has lastOperation, its type is Reconcile and its state is neither Processing or Error", func() { It("should return false", func() { etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeReconcile, State: LastOperationStateSucceeded, } Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) }) }) + Context("when etcd status has lastOperation, and its type is not Reconcile", func() { + It("should return false", func() { + etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeCreate, + } + Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) + }) + }) Context("when etcd status does not have lastOperation populated", func() { It("should return false", func() { Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) }) }) }) + + Context("IsDeletionInProgress", func() { + Context("when etcd status has lastOperation, its type is Delete and its state is Processing", func() { + It("should return true", func() { + etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeDelete, + State: LastOperationStateProcessing, + } + Expect(etcd.IsDeletionInProgress()).To(Equal(true)) + }) + }) + Context("when etcd status has lastOperation, its type is Delete and its state is Error", func() { + It("should return true", func() { + etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeDelete, + State: LastOperationStateError, + } + Expect(etcd.IsDeletionInProgress()).To(Equal(true)) + }) + }) + Context("when etcd status has lastOperation, its type is Delete and its state is neither Processing or Error", func() { + It("should return false", func() { + etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeDelete, + State: LastOperationStateSucceeded, + } + Expect(etcd.IsDeletionInProgress()).To(Equal(false)) + }) + }) + Context("when etcd status has lastOperation, and its type is not Delete", func() { + It("should return false", func() { + etcd.Status.LastOperation = &LastOperation{ + Type: LastOperationTypeCreate, + } + Expect(etcd.IsDeletionInProgress()).To(Equal(false)) + }) + }) + Context("when etcd status does not have lastOperation populated", func() { + It("should return false", func() { + Expect(etcd.IsDeletionInProgress()).To(Equal(false)) + }) + }) + }) }) func getEtcd(name, namespace string) *Etcd { diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index 37ab73686..0276dacb3 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -90,12 +90,17 @@ webhooks: apiVersions: - v1 operations: + - UPDATE - DELETE resources: - leases scope: '*' sideEffects: None timeoutSeconds: 10 + +{{/* This webhook is required for specially handling statefulsets/scale subresource, */}} +{{/* because an `objectSelector` does not work for subresources. */}} +{{/* Refer https://github.com/kubernetes/kubernetes/issues/113594#issuecomment-1332573990. */}} - admissionReviewVersions: - v1beta1 - v1 diff --git a/docs/development/getting-started-locally.md b/docs/development/getting-started-locally.md index 287910953..3ec22c314 100644 --- a/docs/development/getting-started-locally.md +++ b/docs/development/getting-started-locally.md @@ -55,6 +55,11 @@ Either one of these commands may be used to deploy etcd-druid to the configured This generates the `Etcd` and `EtcdCopyBackupsTask` CRDs and deploys an etcd-druid pod into the cluster. +> **Note:** Before calling any of the `make deploy*` commands, certain environment variables may be set in order to enable/disable certain functionalities of etcd-druid. These are: +> - `DRUID_ENABLE_SENTINEL_WEBHOOK=true` : enables the [sentinel webhook](../concepts/webhooks.md#sentinel-webhook) +> - `DRUID_E2E_TEST=true` : sets specific configuration for etcd-druid for optimal e2e test runs, like a lower sync period for the etcd controller. +> - `USE_ETCD_DRUID_FEATURE_GATES=false` : enables etcd-druid feature gates. + ### Prepare the Etcd CR Etcd CR can be configured in 2 ways. Either to take backups to the store or disable them. Follow the appropriate section below based on the requirement. diff --git a/go.mod b/go.mod index 3dd59ef81..a6a7dce85 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 + k8s.io/apiserver v0.28.3 k8s.io/client-go v0.28.3 k8s.io/component-base v0.28.3 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 diff --git a/internal/common/constants.go b/internal/common/constants.go index a277f1e8e..188839bc2 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -106,7 +106,7 @@ const ( // ComponentNameServiceAccount is the component name for service account resource. ComponentNameServiceAccount = "etcd-service-account" // ComponentNameStatefulSet is the component name for statefulset resource. - ComponentNameStatefulSet = "etcd-sts" + ComponentNameStatefulSet = "etcd-statefulset" // ComponentNameCompactionJob is the component name for compaction job resource. ComponentNameCompactionJob = "etcd-compaction-job" // ComponentNameEtcdCopyBackupsJob is the component name for copy-backup task resource. diff --git a/internal/component/statefulset/builder.go b/internal/component/statefulset/builder.go index a239598e8..68a7ce3cf 100644 --- a/internal/component/statefulset/builder.go +++ b/internal/component/statefulset/builder.go @@ -602,7 +602,6 @@ func (b *stsBuilder) getEtcdContainerCommandArgs() []string { return []string{} } commandArgs := []string{"start-etcd"} - // TODO commandArgs = append(commandArgs, fmt.Sprintf("--backup-restore-host-port=%s-local:%d", b.etcd.Name, common.DefaultPortEtcdBackupRestore)) commandArgs = append(commandArgs, fmt.Sprintf("--etcd-server-name=%s-local", b.etcd.Name)) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 64449306a..6f0e718a3 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -9,7 +9,6 @@ import ( "fmt" "net/http" "slices" - "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -25,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/client-go/scale/scheme/autoscalingv1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -62,67 +62,40 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) } - requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} - - obj, err := h.decodeRequestObject(req, requestGK) + etcd, allowedMessage, err := h.getEtcdForRequest(ctx, req) if err != nil { return admission.Errored(http.StatusInternalServerError, err) } - if obj == nil { - return admission.Allowed(fmt.Sprintf("unexpected resource type: %s", requestGKString)) - } - - // If admission request is for statefulsets/scale subresource, then check labels on the parent statefulset object - // and allow the request if it doesn't contain label `app.kubernetes.io/managed-by: etcd-druid`. - // This special handling is required for statefulsets/scale subresource, since it does not work when - // an objectSelector is specified for it in the ValidatingWebhookConfiguration. - // More information can be found at https://github.com/kubernetes/kubernetes/issues/113594#issuecomment-1332573990 - requestResourceGK := schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} - if requestGK == autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind() && - requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() && - req.SubResource == "Scale" { - sts, err := h.fetchStatefulSet(ctx, req.Name, req.Namespace) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - managedBy, hasLabel := sts.GetLabels()[druidv1alpha1.LabelManagedByKey] - if !hasLabel || managedBy != druidv1alpha1.LabelManagedByValue { - return admission.Allowed("statefulset not managed by etcd-druid") - } - // replicate sts labels to the /scale subresource object, used for subsequent checks - obj.SetLabels(sts.GetLabels()) - } - - etcdName, hasLabel := obj.GetLabels()[druidv1alpha1.LabelPartOfKey] - if !hasLabel { - return admission.Allowed(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey)) + if allowedMessage != "" { + return admission.Allowed(allowedMessage) } - etcd := &druidv1alpha1.Etcd{} - if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { - if apierrors.IsNotFound(err) { - return admission.Allowed(fmt.Sprintf("corresponding Etcd %s not found", etcdName)) + // allow deletion operation on resources if the Etcd is currently being deleted, but only by etcd-druid and exempt service accounts. + if req.Operation == admissionv1.Delete && etcd.IsDeletionInProgress() { + if req.UserInfo.Username == h.config.ReconcilerServiceAccount { + return admission.Allowed(fmt.Sprintf("deletion of resource by etcd-druid is allowed during deletion of Etcd %s", etcd.Name)) } - return admission.Errored(http.StatusInternalServerError, err) + if slices.Contains(h.config.ExemptServiceAccounts, req.UserInfo.Username) { + return admission.Allowed(fmt.Sprintf("deletion of resource by exempt SA %s is allowed during deletion of Etcd %s", req.UserInfo.Username, etcd.Name)) + } + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing deletion of Etcd %s by etcd-druid", etcd.Name)) } // Leases (member and snapshot) will be periodically updated by etcd members. // Allow updates to such leases, but only by etcd members, which would use the serviceaccount deployed by druid for them. - if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && - req.Operation == admissionv1.Update { - saTokens := strings.Split(req.UserInfo.Username, ":") - saName := saTokens[len(saTokens)-1] // last element of sa name which looks like `system:serviceaccount::` - if saName == etcd.GetServiceAccountName() { + requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} + if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && req.Operation == admissionv1.Update { + if serviceaccount.MatchesUsername(etcd.GetNamespace(), etcd.GetServiceAccountName(), req.UserInfo.Username) { return admission.Allowed("lease resource can be freely updated by etcd members") } } - // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set + // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set. if !etcd.AreManagedResourcesProtected() { return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } - // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid, + // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid. if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { return admission.Allowed(fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", etcd.Name)) @@ -140,12 +113,48 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", etcd.Name)) } +// getEtcdForRequest returns the Etcd resource corresponding to the object in the admission request. +// It also returns admission response message and error, if any. +func (h *Handler) getEtcdForRequest(ctx context.Context, req admission.Request) (*druidv1alpha1.Etcd, string, error) { + obj, err := h.decodeRequestObject(ctx, req) + if err != nil { + return nil, "", err + } + if obj == nil { + return nil, fmt.Sprintf("unexpected resource type: %s/%s", req.Kind.Group, req.Kind.Kind), nil + } + + // allow changes for resources not managed by etcd-druid + managedBy, hasLabel := obj.GetLabels()[druidv1alpha1.LabelManagedByKey] + if !hasLabel || managedBy != druidv1alpha1.LabelManagedByValue { + return nil, fmt.Sprintf("resource is not managed by etcd-druid, as label %s is missing", druidv1alpha1.LabelManagedByKey), nil + } + + // get the name of the Etcd that the resource is part of + etcdName, hasLabel := obj.GetLabels()[druidv1alpha1.LabelPartOfKey] + if !hasLabel { + return nil, fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), nil + } + + etcd := &druidv1alpha1.Etcd{} + if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Sprintf("corresponding Etcd %s not found", etcdName), nil + } + return nil, "", err + } + + return etcd, "", nil + +} + // decodeRequestObject decodes the relevant object from the admission request and returns it. // If it encounters an unexpected resource type, it returns a nil object. -func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.GroupKind) (client.Object, error) { +func (h *Handler) decodeRequestObject(ctx context.Context, req admission.Request) (client.Object, error) { var ( - obj client.Object - err error + requestGK = schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} + obj client.Object + err error ) switch requestGK { case @@ -155,12 +164,19 @@ func (h *Handler) decodeRequestObject(req admission.Request, requestGK schema.Gr rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(), rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(), appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), - autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(), // for statefulsets' /scale subresource policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): obj, err = h.doDecodeRequestObject(req) + + // if admission request is for statefulsets/scale subresource, then fetch and return the parent statefulset object instead. + case autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(): + requestResourceGK := schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} + if requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() { + obj, err = h.fetchStatefulSet(ctx, req.Name, req.Namespace) + } } + return obj, err } diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index ee1dd28fa..fb706d94a 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -145,7 +145,7 @@ func TestHandleLeaseUpdate(t *testing.T) { } obj := buildObjRawExtension(g, &coordinationv1.Lease{}, nil, testObjectName, testNamespace, - map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}) + map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}) username := testUserName if tc.useEtcdServiceAccount { @@ -206,7 +206,7 @@ func TestHandleStatefulSetScaleSubresourceUpdate(t *testing.T) { }) g.Expect(response.Allowed).To(BeTrue()) - g.Expect(response.Result.Message).To(Equal("statefulset not managed by etcd-druid")) + g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource is not managed by etcd-druid, as label %s is missing", druidv1alpha1.LabelManagedByKey))) g.Expect(response.Result.Code).To(Equal(int32(http.StatusOK))) } @@ -236,6 +236,38 @@ func TestUnexpectedResourceType(t *testing.T) { g.Expect(resp.Result.Message).To(Equal("unexpected resource type: coordination.k8s.io/Unknown")) } +func TestMissingManagedByLabel(t *testing.T) { + g := NewWithT(t) + + cl := fake.NewClientBuilder().Build() + decoder := admission.NewDecoder(cl.Scheme()) + + handler := &Handler{ + Client: cl, + config: &Config{ + Enabled: true, + }, + decoder: decoder, + logger: logr.Discard(), + } + + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}) + response := handler.Handle(context.Background(), admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + UserInfo: authenticationv1.UserInfo{Username: testUserName}, + Kind: statefulSetGVK, + Name: testObjectName, + Namespace: testNamespace, + Object: obj, + OldObject: obj, + }, + }) + + g.Expect(response.Allowed).To(Equal(true)) + g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource is not managed by etcd-druid, as label %s is missing", druidv1alpha1.LabelManagedByKey))) +} + func TestMissingResourcePartOfLabel(t *testing.T) { g := NewWithT(t) @@ -251,7 +283,7 @@ func TestMissingResourcePartOfLabel(t *testing.T) { logger: logr.Discard(), } - obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{}) + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue}) response := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, @@ -291,7 +323,7 @@ func TestHandleUpdate(t *testing.T) { }{ { name: "disable resource protection annotation set", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, expectedAllowed: true, expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), @@ -300,8 +332,8 @@ func TestHandleUpdate(t *testing.T) { { name: "operator makes a request when Etcd is being reconciled by druid", userName: testUserName, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: false, expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", testEtcdName), @@ -310,8 +342,8 @@ func TestHandleUpdate(t *testing.T) { { name: "druid makes a request during its reconciliation run", userName: reconcilerServiceAccount, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: true, expectedMessage: fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", testEtcdName), @@ -320,7 +352,7 @@ func TestHandleUpdate(t *testing.T) { { name: "Etcd is not currently being reconciled by druid, and request is from exempt service account", userName: exemptServiceAccounts[0], - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, @@ -330,7 +362,7 @@ func TestHandleUpdate(t *testing.T) { { name: "Etcd is not currently being reconciled by druid, and request is from non-exempt service account", userName: testUserName, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: false, @@ -506,7 +538,10 @@ func TestEtcdGetFailures(t *testing.T) { logger: logr.Discard(), } - obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}) + obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{ + druidv1alpha1.LabelPartOfKey: testEtcdName, + druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, + }) response := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ @@ -550,17 +585,17 @@ func TestHandleDelete(t *testing.T) { }{ { name: "disable resource protection annotation set", - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, expectedAllowed: true, expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), expectedCode: http.StatusOK, }, { - name: "Etcd is currently being reconciled by druid", + name: "Etcd is currently being reconciled by druid, and request is from non-exempt service account", userName: testUserName, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: false, expectedReason: "Forbidden", @@ -568,19 +603,31 @@ func TestHandleDelete(t *testing.T) { expectedCode: http.StatusForbidden, }, { - name: "Etcd is currently being reconciled by druid, but request is from druid", + name: "Etcd is currently being reconciled by druid, and request is from druid", userName: reconcilerServiceAccount, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdStatusLastOperation: &druidv1alpha1.LastOperation{State: druidv1alpha1.LastOperationStateProcessing}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateProcessing}, reconcilerServiceAccount: reconcilerServiceAccount, expectedAllowed: true, expectedMessage: fmt.Sprintf("ongoing reconciliation of Etcd %s by etcd-druid requires changes to resources", testEtcdName), expectedCode: http.StatusOK, }, + { + name: "Etcd is currently being reconciled by druid, and request is from exempt service account", + userName: exemptServiceAccounts[0], + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateProcessing}, + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + expectedAllowed: false, + expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing reconciliation of Etcd %s by etcd-druid", testEtcdName), + expectedReason: "Forbidden", + expectedCode: http.StatusForbidden, + }, { name: "Etcd is not currently being reconciled by druid, and request is from exempt service account", userName: exemptServiceAccounts[0], - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, @@ -588,9 +635,9 @@ func TestHandleDelete(t *testing.T) { expectedCode: http.StatusOK, }, { - name: "Etcd is not currently being reconciled by druid, and request is from non-exempt service account", + name: "Etcd is not currently being reconciled or deleted by druid, and request is from non-exempt service account", userName: testUserName, - objectLabels: map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: false, @@ -598,6 +645,38 @@ func TestHandleDelete(t *testing.T) { expectedReason: "Forbidden", expectedCode: http.StatusForbidden, }, + { + name: "Etcd is currently being deleted by druid, and request is from non-exempt service account", + userName: testUserName, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeDelete, State: druidv1alpha1.LastOperationStateProcessing}, + reconcilerServiceAccount: reconcilerServiceAccount, + expectedAllowed: false, + expectedReason: "Forbidden", + expectedMessage: fmt.Sprintf("no external intervention allowed during ongoing deletion of Etcd %s by etcd-druid", testEtcdName), + expectedCode: http.StatusForbidden, + }, + { + name: "Etcd is currently being deleted by druid, and request is from druid", + userName: reconcilerServiceAccount, + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeDelete, State: druidv1alpha1.LastOperationStateProcessing}, + reconcilerServiceAccount: reconcilerServiceAccount, + expectedAllowed: true, + expectedMessage: fmt.Sprintf("deletion of resource by etcd-druid is allowed during deletion of Etcd %s", testEtcdName), + expectedCode: http.StatusOK, + }, + { + name: "Etcd is not currently being deleted by druid, and request is from exempt service account", + userName: exemptServiceAccounts[0], + objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, + etcdStatusLastOperation: &druidv1alpha1.LastOperation{Type: druidv1alpha1.LastOperationTypeDelete, State: druidv1alpha1.LastOperationStateProcessing}, + reconcilerServiceAccount: reconcilerServiceAccount, + exemptServiceAccounts: exemptServiceAccounts, + expectedAllowed: true, + expectedMessage: fmt.Sprintf("deletion of resource by exempt SA %s is allowed during deletion of Etcd %s", exemptServiceAccounts[0], testEtcdName), + expectedCode: http.StatusOK, + }, } for _, tc := range testCases { diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index d85f59107..324069c49 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -200,9 +200,10 @@ func testEtcdSpecUpdateWhenNoReconcileOperationAnnotationIsSet(t *testing.T, tes // get updated version of etcdInstance g.Expect(cl.Get(ctx, etcdInstance.GetNamespaceName(), etcdInstance)).To(Succeed()) // update etcdInstance spec without reconcile operation annotation set + originalEtcdInstance := etcdInstance.DeepCopy() metricsLevelExtensive := druidv1alpha1.Extensive etcdInstance.Spec.Etcd = druidv1alpha1.EtcdConfig{Metrics: &metricsLevelExtensive} - g.Expect(cl.Update(ctx, etcdInstance)).To(Succeed()) + g.Expect(cl.Patch(ctx, etcdInstance, client.MergeFrom(originalEtcdInstance))).To(Succeed()) // ***************** test etcd spec reconciliation ***************** assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 2*time.Second, 2*time.Second) @@ -227,12 +228,13 @@ func testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet(t *testing.T, testN // get latest version of etcdInstance g.Expect(cl.Get(ctx, etcdInstance.GetNamespaceName(), etcdInstance)).To(Succeed()) // update etcdInstance spec with reconcile operation annotation also set + originalEtcdInstance := etcdInstance.DeepCopy() metricsLevelExtensive := druidv1alpha1.Extensive etcdInstance.Spec.Etcd = druidv1alpha1.EtcdConfig{Metrics: &metricsLevelExtensive} etcdInstance.Annotations = map[string]string{ v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, } - g.Expect(cl.Update(ctx, etcdInstance)).To(Succeed()) + g.Expect(cl.Patch(ctx, etcdInstance, client.MergeFrom(originalEtcdInstance))).To(Succeed()) // ***************** test etcd spec reconciliation ***************** assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 30*time.Second, 2*time.Second) From 6d87c3e4fd11849fae92e15eb278f99e5d68a334 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 22 May 2024 17:49:29 +0530 Subject: [PATCH 191/235] Address review comments by @ashwani2k and @renormalize: abstract out individual controllers and webhooks from manager code --- internal/controller/compaction/config.go | 2 +- internal/controller/config.go | 68 +++++++++++++++ internal/controller/etcd/config.go | 3 +- .../controller/etcdcopybackupstask/config.go | 2 +- internal/controller/register.go | 59 +++++++++++++ internal/controller/secret/config.go | 2 +- internal/manager/config.go | 82 ++++--------------- internal/manager/manager.go | 72 ++-------------- internal/webhook/config.go | 33 ++++++++ internal/webhook/register.go | 25 ++++++ 10 files changed, 211 insertions(+), 137 deletions(-) create mode 100644 internal/controller/config.go create mode 100644 internal/controller/register.go create mode 100644 internal/webhook/config.go create mode 100644 internal/webhook/register.go diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index d0d5f40bc..9d6fde80c 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -50,7 +50,7 @@ type Config struct { } // InitFromFlags initializes the compaction controller config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { fs.BoolVar(&cfg.EnableBackupCompaction, enableBackupCompactionFlagName, defaultEnableBackupCompaction, "Enable automatic compaction of etcd backups.") fs.IntVar(&cfg.Workers, workersFlagName, defaultCompactionWorkers, diff --git a/internal/controller/config.go b/internal/controller/config.go new file mode 100644 index 000000000..25a4ed2df --- /dev/null +++ b/internal/controller/config.go @@ -0,0 +1,68 @@ +package controller + +import ( + "github.com/gardener/etcd-druid/internal/controller/compaction" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" + "github.com/gardener/etcd-druid/internal/controller/secret" + + flag "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +// Config defines the configuration for etcd-druid controllers. +type Config struct { + // Etcd is the configuration required for etcd controller. + Etcd *etcd.Config + // Compaction is the configuration required for compaction controller. + Compaction *compaction.Config + // EtcdCopyBackupsTask is the configuration required for etcd-copy-backup-tasks controller. + EtcdCopyBackupsTask *etcdcopybackupstask.Config + // Secret is the configuration required for secret controller. + Secret *secret.Config +} + +// InitFromFlags initializes the controller config from the provided CLI flag set. +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { + cfg.Etcd = &etcd.Config{} + cfg.Etcd.InitFromFlags(fs) + + cfg.Compaction = &compaction.Config{} + cfg.Compaction.InitFromFlags(fs) + + cfg.EtcdCopyBackupsTask = &etcdcopybackupstask.Config{} + cfg.EtcdCopyBackupsTask.InitFromFlags(fs) + + cfg.Secret = &secret.Config{} + cfg.Secret.InitFromFlags(fs) +} + +// Validate validates the controller config. +func (cfg *Config) Validate() error { + if err := cfg.Etcd.Validate(); err != nil { + return err + } + + if err := cfg.Compaction.Validate(); err != nil { + return err + } + + if err := cfg.EtcdCopyBackupsTask.Validate(); err != nil { + return err + } + + return cfg.Secret.Validate() +} + +// CaptureFeatureActivations captures feature gate activations for every controller configuration. +// Feature gates are captured only for controllers that use feature gates. +func (cfg *Config) CaptureFeatureActivations(featureGates featuregate.MutableFeatureGate) { + // Add etcd controller feature gates + cfg.Etcd.CaptureFeatureActivations(featureGates) + + // Add compaction controller feature gates + cfg.Compaction.CaptureFeatureActivations(featureGates) + + // Add etcd-copy-backups-task controller feature gates + cfg.EtcdCopyBackupsTask.CaptureFeatureActivations(featureGates) +} diff --git a/internal/controller/etcd/config.go b/internal/controller/etcd/config.go index 93cd74d8b..5ff97434d 100644 --- a/internal/controller/etcd/config.go +++ b/internal/controller/etcd/config.go @@ -11,6 +11,7 @@ import ( "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" + gardenerconstants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" flag "github.com/spf13/pflag" "k8s.io/component-base/featuregate" @@ -77,7 +78,7 @@ type MemberConfig struct { } // InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, "Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed.") flag.BoolVar(&cfg.IgnoreOperationAnnotation, ignoreOperationAnnotationFlagName, defaultIgnoreOperationAnnotation, diff --git a/internal/controller/etcdcopybackupstask/config.go b/internal/controller/etcdcopybackupstask/config.go index 66ff9edfd..74d9c184b 100644 --- a/internal/controller/etcdcopybackupstask/config.go +++ b/internal/controller/etcdcopybackupstask/config.go @@ -32,7 +32,7 @@ type Config struct { } // InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, "Number of worker threads for the etcdcopybackupstask controller.") } diff --git a/internal/controller/register.go b/internal/controller/register.go new file mode 100644 index 000000000..9ff297183 --- /dev/null +++ b/internal/controller/register.go @@ -0,0 +1,59 @@ +package controller + +import ( + "context" + "time" + + "github.com/gardener/etcd-druid/internal/controller/compaction" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" + "github.com/gardener/etcd-druid/internal/controller/secret" + + ctrl "sigs.k8s.io/controller-runtime" +) + +var ( + defaultTimeout = time.Minute +) + +// Register registers all etcd-druid controllers with the controller manager. +func Register(mgr ctrl.Manager, config *Config) error { + var err error + + // Add etcd reconciler to the manager + etcdReconciler, err := etcd.NewReconciler(mgr, config.Etcd) + if err != nil { + return err + } + if err = etcdReconciler.RegisterWithManager(mgr); err != nil { + return err + } + + // Add compaction reconciler to the manager if the CLI flag enable-backup-compaction is true. + if config.Compaction.EnableBackupCompaction { + compactionReconciler, err := compaction.NewReconciler(mgr, config.Compaction) + if err != nil { + return err + } + if err = compactionReconciler.RegisterWithManager(mgr); err != nil { + return err + } + } + + // Add etcd-copy-backups-task reconciler to the manager + etcdCopyBackupsTaskReconciler, err := etcdcopybackupstask.NewReconciler(mgr, config.EtcdCopyBackupsTask) + if err != nil { + return err + } + if err = etcdCopyBackupsTaskReconciler.RegisterWithManager(mgr); err != nil { + return err + } + + // Add secret reconciler to the manager + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + return secret.NewReconciler( + mgr, + config.Secret, + ).RegisterWithManager(ctx, mgr) +} diff --git a/internal/controller/secret/config.go b/internal/controller/secret/config.go index c45c95631..63e4c5172 100644 --- a/internal/controller/secret/config.go +++ b/internal/controller/secret/config.go @@ -23,7 +23,7 @@ type Config struct { } // InitFromFlags initializes the config from the provided CLI flag set. -func InitFromFlags(fs *flag.FlagSet, cfg *Config) { +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.Workers, workersFlagName, defaultWorkers, "Number of worker threads for the secrets controller.") } diff --git a/internal/manager/config.go b/internal/manager/config.go index 11281c2de..292bff3c1 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -7,13 +7,10 @@ package controller import ( "fmt" - "github.com/gardener/etcd-druid/internal/controller/compaction" - "github.com/gardener/etcd-druid/internal/controller/etcd" - "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" - "github.com/gardener/etcd-druid/internal/controller/secret" + "github.com/gardener/etcd-druid/internal/controller" "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/features" - "github.com/gardener/etcd-druid/internal/webhook/sentinel" + "github.com/gardener/etcd-druid/internal/webhook" flag "github.com/spf13/pflag" "k8s.io/client-go/tools/leaderelection/resourcelock" @@ -100,33 +97,9 @@ type Config struct { // FeatureGates contains the feature gates to be used by etcd-druid. FeatureGates featuregate.MutableFeatureGate // Controllers defines the configuration for etcd-druid controllers. - Controllers Controllers + Controllers *controller.Config // Webhooks defines the configuration for etcd-druid webhooks. - Webhooks Webhooks -} - -// Controllers defines the configuration for etcd druid controllers. -type Controllers struct { - // Etcd is the configuration required for etcd controller. - Etcd *etcd.Config - // Compaction is the configuration required for compaction controller. - Compaction *compaction.Config - // EtcdCopyBackupsTask is the configuration required for etcd-copy-backup-tasks controller. - EtcdCopyBackupsTask *etcdcopybackupstask.Config - // Secret is the configuration required for secret controller. - Secret *secret.Config -} - -// Webhooks defines the configuration for etcd-druid webhooks. -type Webhooks struct { - // Sentinel is the configuration required for sentinel webhook. - Sentinel *sentinel.Config -} - -// AtLeaseOneEnabled returns true if at least one webhook is enabled. -// NOTE for contributors: For every new webhook, add a disjunction condition with the webhook's AtLeaseOneEnabled field. -func (w Webhooks) AtLeaseOneEnabled() bool { - return w.Sentinel.Enabled + Webhooks *webhook.Config } // InitFromFlags initializes the controller manager config from the provided CLI flag set. @@ -161,20 +134,11 @@ func (cfg *Config) InitFromFlags(fs *flag.FlagSet) error { return err } - cfg.Controllers.Etcd = &etcd.Config{} - etcd.InitFromFlags(fs, cfg.Controllers.Etcd) - - cfg.Controllers.Compaction = &compaction.Config{} - compaction.InitFromFlags(fs, cfg.Controllers.Compaction) - - cfg.Controllers.EtcdCopyBackupsTask = &etcdcopybackupstask.Config{} - etcdcopybackupstask.InitFromFlags(fs, cfg.Controllers.EtcdCopyBackupsTask) - - cfg.Controllers.Secret = &secret.Config{} - secret.InitFromFlags(fs, cfg.Controllers.Secret) + cfg.Controllers = &controller.Config{} + cfg.Controllers.InitFromFlags(fs) - cfg.Webhooks.Sentinel = &sentinel.Config{} - sentinel.InitFromFlags(fs, cfg.Webhooks.Sentinel) + cfg.Webhooks = &webhook.Config{} + cfg.Webhooks.InitFromFlags(fs) return nil } @@ -192,18 +156,10 @@ func (cfg *Config) initFeatureGates(fs *flag.FlagSet) error { return nil } -// populateControllersFeatureGates adds relevant feature gates to every controller configuration -func (cfg *Config) populateControllersFeatureGates() { - // Feature gates populated only for controllers that use feature gates - - // Add etcd controller feature gates - cfg.Controllers.Etcd.CaptureFeatureActivations(cfg.FeatureGates) - - // Add compaction controller feature gates - cfg.Controllers.Compaction.CaptureFeatureActivations(cfg.FeatureGates) - - // Add etcd-copy-backups-task controller feature gates - cfg.Controllers.EtcdCopyBackupsTask.CaptureFeatureActivations(cfg.FeatureGates) +// captureFeatureActivations captures feature gate activations for etcd-druid controllers and webhooks. +func (cfg *Config) captureFeatureActivations() { + cfg.Controllers.CaptureFeatureActivations(cfg.FeatureGates) + cfg.Webhooks.CaptureFeatureActivations(cfg.FeatureGates) } // Validate validates the controller manager config. @@ -212,7 +168,7 @@ func (cfg *Config) Validate() error { return err } - if cfg.Webhooks.Sentinel.Enabled { + if cfg.Webhooks.AtLeaseOneEnabled() { if cfg.Server.Webhook.Port == 0 { return fmt.Errorf("webhook port cannot be 0") } @@ -221,19 +177,11 @@ func (cfg *Config) Validate() error { } } - if err := cfg.Controllers.Etcd.Validate(); err != nil { - return err - } - - if err := cfg.Controllers.Compaction.Validate(); err != nil { - return err - } - - if err := cfg.Controllers.EtcdCopyBackupsTask.Validate(); err != nil { + if err := cfg.Controllers.Validate(); err != nil { return err } - return cfg.Controllers.Secret.Validate() + return cfg.Webhooks.Validate() } // getAllowedLeaderElectionResourceLocks returns the allowed resource type to be used for leader election. diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 23cad89d0..22e54ac17 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -5,19 +5,16 @@ package controller import ( - "context" "net" "net/http" "strconv" "strings" "time" + druidcontroller "github.com/gardener/etcd-druid/internal/controller" + druidwebhook "github.com/gardener/etcd-druid/internal/webhook" + "github.com/gardener/etcd-druid/internal/client/kubernetes" - "github.com/gardener/etcd-druid/internal/controller/compaction" - "github.com/gardener/etcd-druid/internal/controller/etcd" - "github.com/gardener/etcd-druid/internal/controller/etcdcopybackupstask" - "github.com/gardener/etcd-druid/internal/controller/secret" - "github.com/gardener/etcd-druid/internal/webhook/sentinel" "golang.org/x/exp/slog" coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" @@ -40,16 +37,16 @@ func InitializeManager(config *Config) (ctrl.Manager, error) { err error mgr ctrl.Manager ) - config.populateControllersFeatureGates() + config.captureFeatureActivations() if mgr, err = createManager(config); err != nil { return nil, err } slog.Info("registering controllers and webhooks with manager") time.Sleep(10 * time.Second) - if err = registerControllers(mgr, config); err != nil { + if err = druidcontroller.Register(mgr, config.Controllers); err != nil { return nil, err } - if err = registerWebhooks(mgr, config); err != nil { + if err = druidwebhook.Register(mgr, config.Webhooks); err != nil { return nil, err } if err = registerHealthAndReadyEndpoints(mgr, config); err != nil { @@ -97,63 +94,6 @@ func createManager(config *Config) (ctrl.Manager, error) { }) } -func registerControllers(mgr ctrl.Manager, config *Config) error { - var err error - - // Add etcd reconciler to the manager - etcdReconciler, err := etcd.NewReconciler(mgr, config.Controllers.Etcd) - if err != nil { - return err - } - if err = etcdReconciler.RegisterWithManager(mgr); err != nil { - return err - } - - // Add compaction reconciler to the manager if the CLI flag enable-backup-compaction is true. - if config.Controllers.Compaction.EnableBackupCompaction { - compactionReconciler, err := compaction.NewReconciler(mgr, config.Controllers.Compaction) - if err != nil { - return err - } - if err = compactionReconciler.RegisterWithManager(mgr); err != nil { - return err - } - } - - // Add etcd-copy-backups-task reconciler to the manager - etcdCopyBackupsTaskReconciler, err := etcdcopybackupstask.NewReconciler(mgr, config.Controllers.EtcdCopyBackupsTask) - if err != nil { - return err - } - if err = etcdCopyBackupsTaskReconciler.RegisterWithManager(mgr); err != nil { - return err - } - - // Add secret reconciler to the manager - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) - defer cancel() - return secret.NewReconciler( - mgr, - config.Controllers.Secret, - ).RegisterWithManager(ctx, mgr) -} - -func registerWebhooks(mgr ctrl.Manager, config *Config) error { - // Add sentinel webhook to the manager - if config.Webhooks.Sentinel.Enabled { - sentinelWebhook, err := sentinel.NewHandler( - mgr, - config.Webhooks.Sentinel, - ) - if err != nil { - return err - } - slog.Info("Registering Sentinel Webhook with manager") - return sentinelWebhook.RegisterWithManager(mgr) - } - return nil -} - func registerHealthAndReadyEndpoints(mgr ctrl.Manager, config *Config) error { slog.Info("Registering ping health check endpoint") if err := mgr.AddHealthzCheck("ping", func(req *http.Request) error { return nil }); err != nil { diff --git a/internal/webhook/config.go b/internal/webhook/config.go new file mode 100644 index 000000000..1b20ac561 --- /dev/null +++ b/internal/webhook/config.go @@ -0,0 +1,33 @@ +package webhook + +import ( + "github.com/gardener/etcd-druid/internal/webhook/sentinel" + + flag "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +// Config defines the configuration for etcd-druid webhooks. +type Config struct { + // Sentinel is the configuration required for sentinel webhook. + Sentinel *sentinel.Config +} + +// InitFromFlags initializes the webhook config from the provided CLI flag set. +func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { + cfg.Sentinel = &sentinel.Config{} + sentinel.InitFromFlags(fs, cfg.Sentinel) +} + +// Validate validates the webhook config. +func (cfg *Config) Validate() error { return nil } + +// CaptureFeatureActivations captures feature gate activations for every webhook configuration. +// Feature gates are captured only for webhooks that use feature gates. +func (cfg *Config) CaptureFeatureActivations(_ featuregate.MutableFeatureGate) {} + +// AtLeaseOneEnabled returns true if at least one webhook is enabled. +// NOTE for contributors: For every new webhook, add a disjunction condition with the webhook's Enabled field. +func (cfg *Config) AtLeaseOneEnabled() bool { + return cfg.Sentinel.Enabled +} diff --git a/internal/webhook/register.go b/internal/webhook/register.go new file mode 100644 index 000000000..635e977f6 --- /dev/null +++ b/internal/webhook/register.go @@ -0,0 +1,25 @@ +package webhook + +import ( + "github.com/gardener/etcd-druid/internal/webhook/sentinel" + + "golang.org/x/exp/slog" + ctrl "sigs.k8s.io/controller-runtime" +) + +// Register registers all etcd-druid webhooks with the controller manager. +func Register(mgr ctrl.Manager, config *Config) error { + // Add sentinel webhook to the manager + if config.Sentinel.Enabled { + sentinelWebhook, err := sentinel.NewHandler( + mgr, + config.Sentinel, + ) + if err != nil { + return err + } + slog.Info("Registering Sentinel Webhook with manager") + return sentinelWebhook.RegisterWithManager(mgr) + } + return nil +} From 48ccfb40b71fbf848a1cff32616fbdc67eb9fb7c Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 09:14:57 +0530 Subject: [PATCH 192/235] component.Operator changed to use metav1.ObjectMeta, removed ginkgo from api package, introduced helper functions in api, renamed files in api package --- Makefile | 5 +- .../{types_common.go => conditions.go} | 24 +- .../{types_constant.go => constants.go} | 0 api/v1alpha1/{types_etcd.go => etcd.go} | 115 ----- api/v1alpha1/etcd_test.go | 227 ++++++++++ api/v1alpha1/helper.go | 128 ++++++ api/v1alpha1/helper_test.go | 246 +++++++++++ .../{groupversion_info.go => register.go} | 7 +- api/v1alpha1/store.go | 22 + api/v1alpha1/suite_test.go | 30 -- api/v1alpha1/types_etcd_test.go | 413 ------------------ .../10-crd-druid.gardener.cloud_etcds.yaml | 80 +--- .../10-crd-druid.gardener.cloud_etcds.yaml | 80 +--- .../component/clientservice/clientservice.go | 30 +- .../clientservice/clientservice_test.go | 30 +- internal/component/configmap/configmap.go | 32 +- .../component/configmap/configmap_test.go | 34 +- internal/component/configmap/etcdconfig.go | 10 +- internal/component/memberlease/memberlease.go | 30 +- .../component/memberlease/memberlease_test.go | 16 +- internal/component/peerservice/peerservice.go | 30 +- .../component/peerservice/peerservice_test.go | 25 +- .../poddisruptionbudget.go | 28 +- .../poddisruptionbudget_test.go | 22 +- internal/component/role/role.go | 28 +- internal/component/role/role_test.go | 22 +- internal/component/rolebinding/rolebinding.go | 32 +- .../component/rolebinding/rolebinding_test.go | 28 +- .../serviceaccount/serviceaccount.go | 28 +- .../serviceaccount/serviceaccount_test.go | 22 +- .../component/snapshotlease/snapshotlease.go | 44 +- .../snapshotlease/snapshotlease_test.go | 22 +- internal/component/statefulset/builder.go | 24 +- internal/component/statefulset/statefulset.go | 54 +-- .../component/statefulset/statefulset_test.go | 8 +- internal/component/statefulset/stsmatcher.go | 8 +- internal/component/types.go | 5 +- internal/controller/compaction/config.go | 7 +- internal/controller/compaction/reconciler.go | 34 +- internal/controller/etcd/reconcile_delete.go | 30 +- internal/controller/etcd/reconcile_spec.go | 28 +- internal/controller/etcd/reconcile_status.go | 2 +- internal/controller/etcd/reconciler.go | 27 +- internal/controller/utils/reconciler.go | 19 + .../check_data_volumes_ready_test.go | 2 +- internal/manager/manager.go | 15 + internal/utils/statefulset.go | 2 +- internal/utils/statefulset_test.go | 3 +- internal/webhook/sentinel/handler.go | 13 + internal/webhook/sentinel/handler_test.go | 2 +- 50 files changed, 1068 insertions(+), 1105 deletions(-) rename api/v1alpha1/{types_common.go => conditions.go} (65%) rename api/v1alpha1/{types_constant.go => constants.go} (100%) rename api/v1alpha1/{types_etcd.go => etcd.go} (84%) create mode 100644 api/v1alpha1/etcd_test.go create mode 100644 api/v1alpha1/helper.go create mode 100644 api/v1alpha1/helper_test.go rename api/v1alpha1/{groupversion_info.go => register.go} (77%) create mode 100644 api/v1alpha1/store.go delete mode 100644 api/v1alpha1/suite_test.go delete mode 100644 api/v1alpha1/types_etcd_test.go diff --git a/Makefile b/Makefile index dfd312291..ec2a8eec6 100644 --- a/Makefile +++ b/Makefile @@ -79,15 +79,14 @@ generate: manifests $(CONTROLLER_GEN) $(GOIMPORTS) $(MOCKGEN) .PHONY: test test: $(GINKGO) $(GOTESTFMT) # run ginkgo unit tests. These will be ported to golang native tests over a period of time. - @"$(HACK_DIR)/test.sh" ./api/... \ - ./internal/controller/etcdcopybackupstask/... \ + @"$(HACK_DIR)/test.sh" ./internal/controller/etcdcopybackupstask/... \ ./internal/controller/predicate/... \ ./internal/controller/secret/... \ ./internal/controller/utils/... \ ./internal/mapper/... \ ./internal/metrics/... # run the golang native unit tests. - @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./internal/controller/etcd/... ./internal/component/... ./internal/utils/... ./internal/webhook/... + @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./api/... ./internal/controller/etcd/... ./internal/component/... ./internal/utils/... ./internal/webhook/... .PHONY: test-integration test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) diff --git a/api/v1alpha1/types_common.go b/api/v1alpha1/conditions.go similarity index 65% rename from api/v1alpha1/types_common.go rename to api/v1alpha1/conditions.go index 301bc1090..e9f7888b4 100644 --- a/api/v1alpha1/types_common.go +++ b/api/v1alpha1/conditions.go @@ -5,30 +5,10 @@ package v1alpha1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// StorageProvider defines the type of object store provider for storing backups. -type StorageProvider string - -// StoreSpec defines parameters related to ObjectStore persisting backups -type StoreSpec struct { - // Container is the name of the container the backup is stored at. - // +optional - Container *string `json:"container,omitempty"` - // Prefix is the prefix used for the store. - // +required - Prefix string `json:"prefix"` - // Provider is the name of the backup provider. - // +optional - Provider *StorageProvider `json:"provider,omitempty"` - // SecretRef is the reference to the secret which used to connect to the backup store. - // +optional - SecretRef *corev1.SecretReference `json:"secretRef,omitempty"` -} - -// ConditionType is the type of a condition. +// ConditionType is the type of condition. type ConditionType string // ConditionStatus is the status of a condition. @@ -60,6 +40,6 @@ type Condition struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` // The reason for the condition's last transition. Reason string `json:"reason"` - // A human readable message indicating details about the transition. + // A human-readable message indicating details about the transition. Message string `json:"message"` } diff --git a/api/v1alpha1/types_constant.go b/api/v1alpha1/constants.go similarity index 100% rename from api/v1alpha1/types_constant.go rename to api/v1alpha1/constants.go diff --git a/api/v1alpha1/types_etcd.go b/api/v1alpha1/etcd.go similarity index 84% rename from api/v1alpha1/types_etcd.go rename to api/v1alpha1/etcd.go index d6cd9e2e4..366282058 100644 --- a/api/v1alpha1/types_etcd.go +++ b/api/v1alpha1/etcd.go @@ -5,13 +5,9 @@ package v1alpha1 import ( - "fmt" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" ) const ( @@ -480,122 +476,11 @@ type LastError struct { ObservedAt metav1.Time `json:"observedAt"` } -// GetNamespaceName is a convenience function which creates a types.NamespacedName for an Etcd resource. -func (e *Etcd) GetNamespaceName() types.NamespacedName { - return types.NamespacedName{ - Namespace: e.Namespace, - Name: e.Name, - } -} - -// GetPeerServiceName returns the peer service name for the Etcd cluster reachable by members within the Etcd cluster. -func (e *Etcd) GetPeerServiceName() string { - return fmt.Sprintf("%s-peer", e.Name) -} - -// GetClientServiceName returns the client service name for the Etcd cluster reachable by external clients. -func (e *Etcd) GetClientServiceName() string { - return fmt.Sprintf("%s-client", e.Name) -} - -// GetServiceAccountName returns the service account name for the Etcd. -func (e *Etcd) GetServiceAccountName() string { - return e.Name -} - -// GetConfigMapName returns the name of the configmap for the Etcd. -func (e *Etcd) GetConfigMapName() string { - return fmt.Sprintf("etcd-bootstrap-%s", string(e.UID[:6])) -} - -// GetCompactionJobName returns the compaction job name for the Etcd. -func (e *Etcd) GetCompactionJobName() string { - return fmt.Sprintf("%s-compactor", e.Name) -} - -// GetOrdinalPodName returns the Etcd pod name based on the ordinal. -func (e *Etcd) GetOrdinalPodName(ordinal int) string { - return fmt.Sprintf("%s-%d", e.Name, ordinal) -} - -// GetDeltaSnapshotLeaseName returns the name of the delta snapshot lease for the Etcd. -func (e *Etcd) GetDeltaSnapshotLeaseName() string { - return fmt.Sprintf("%s-delta-snap", e.Name) -} - -// GetFullSnapshotLeaseName returns the name of the full snapshot lease for the Etcd. -func (e *Etcd) GetFullSnapshotLeaseName() string { - return fmt.Sprintf("%s-full-snap", e.Name) -} - -// GetMemberLeaseNames returns the name of member leases for the Etcd. -func (e *Etcd) GetMemberLeaseNames() []string { - numReplicas := int(e.Spec.Replicas) - leaseNames := make([]string, 0, numReplicas) - for i := 0; i < numReplicas; i++ { - leaseNames = append(leaseNames, fmt.Sprintf("%s-%d", e.Name, i)) - } - return leaseNames -} - -// GetDefaultLabels returns the default labels for etcd. -func (e *Etcd) GetDefaultLabels() map[string]string { - return map[string]string{ - LabelManagedByKey: LabelManagedByValue, - LabelPartOfKey: e.Name, - } -} - -// GetAsOwnerReference returns an OwnerReference object that represents the current Etcd instance. -func (e *Etcd) GetAsOwnerReference() metav1.OwnerReference { - return metav1.OwnerReference{ - APIVersion: GroupVersion.String(), - Kind: "Etcd", - Name: e.Name, - UID: e.UID, - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - } -} - -// GetRoleName returns the role name for the Etcd -func (e *Etcd) GetRoleName() string { - return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, e.Name) -} - -// GetRoleBindingName returns the rolebinding name for the Etcd -func (e *Etcd) GetRoleBindingName() string { - return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, e.Name) -} - // IsBackupStoreEnabled returns true if backup store has been enabled for the Etcd resource, else returns false. func (e *Etcd) IsBackupStoreEnabled() bool { return e.Spec.Backup.Store != nil } -// IsMarkedForDeletion returns true if a deletion timestamp has been set and false otherwise. -func (e *Etcd) IsMarkedForDeletion() bool { - return !e.DeletionTimestamp.IsZero() -} - -// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an Etcd resource signalling the intent -// to suspend spec reconciliation for this Etcd resource. If no annotation is set then it will return nil. -func (e *Etcd) GetSuspendEtcdSpecReconcileAnnotationKey() *string { - if metav1.HasAnnotation(e.ObjectMeta, SuspendEtcdSpecReconcileAnnotation) { - return pointer.String(SuspendEtcdSpecReconcileAnnotation) - } - if metav1.HasAnnotation(e.ObjectMeta, IgnoreReconciliationAnnotation) { - return pointer.String(IgnoreReconciliationAnnotation) - } - return nil -} - -// AreManagedResourcesProtected returns false if the Etcd resource has the `druid.gardener.cloud/disable-resource-protection` annotation set, -// else returns true. -func (e *Etcd) AreManagedResourcesProtected() bool { - return !metav1.HasAnnotation(e.ObjectMeta, DisableResourceProtectionAnnotation) -} - // IsReconciliationInProgress returns true if the Etcd resource is currently being reconciled, else returns false. func (e *Etcd) IsReconciliationInProgress() bool { return e.Status.LastOperation != nil && diff --git a/api/v1alpha1/etcd_test.go b/api/v1alpha1/etcd_test.go new file mode 100644 index 000000000..823360abb --- /dev/null +++ b/api/v1alpha1/etcd_test.go @@ -0,0 +1,227 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1_test + +import ( + "testing" + "time" + + . "github.com/gardener/etcd-druid/api/v1alpha1" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// These tests are written in BDD-style using Ginkgo framework. Refer to +// http://onsi.github.io/ginkgo to learn more. + +func TestIsBackupStoreEnabled(t *testing.T) { + etcd := createEtcd("foo", "default") + tests := []struct { + name string + backup BackupSpec + expected bool + }{ + { + name: "when backup is enabled", + backup: BackupSpec{Store: &StoreSpec{}}, + expected: true, + }, + { + name: "when backup is not enabled", + backup: BackupSpec{}, + expected: false, + }, + } + g := NewWithT(t) + t.Parallel() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etcd.Spec.Backup = test.backup + g.Expect(etcd.IsBackupStoreEnabled()).To(Equal(test.expected)) + }) + } + +} + +func TestIsReconciliationInProgress(t *testing.T) { + tests := []struct { + name string + lastOp *LastOperation + expected bool + }{ + { + name: "when etcd status has lastOperation set and its state is Processing", + lastOp: &LastOperation{ + State: LastOperationStateProcessing, + }, + expected: true, + }, + { + name: "when etcd status has lastOperation set and its state is Error", + lastOp: &LastOperation{ + State: LastOperationStateError, + }, + expected: true, + }, + { + name: "when etcd status has lastOperation set and its state is Succeeded", + lastOp: &LastOperation{ + State: LastOperationStateSucceeded, + }, + expected: false, + }, + { + name: "when etcd status does not have lastOperation set", + lastOp: nil, + expected: false, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etcd := createEtcd("foo", "default") + etcd.Status.LastOperation = test.lastOp + g.Expect(etcd.IsReconciliationInProgress()).To(Equal(test.expected)) + }) + } +} + +func createEtcd(name, namespace string) *Etcd { + var ( + clientPort int32 = 2379 + serverPort int32 = 2380 + backupPort int32 = 8080 + metricLevel = Basic + ) + + garbageCollectionPeriod := metav1.Duration{ + Duration: 43200 * time.Second, + } + deltaSnapshotPeriod := metav1.Duration{ + Duration: 300 * time.Second, + } + imageEtcd := "europe-docker.pkg.dev/gardener-project/public/gardener/etcd-wrapper:v0.1.0" + imageBR := "europe-docker.pkg.dev/gardener-project/public/gardener/etcdbrctl:v0.25.0" + snapshotSchedule := "0 */24 * * *" + defragSchedule := "0 */24 * * *" + container := "my-object-storage-container-name" + storageCapacity := resource.MustParse("20Gi") + deltaSnapShotMemLimit := resource.MustParse("100Mi") + quota := resource.MustParse("8Gi") + storageClass := "gardener.cloud-fast" + provider := StorageProvider("aws") + prefix := "etcd-test" + garbageCollectionPolicy := GarbageCollectionPolicy(GarbageCollectionPolicyExponential) + + clientTlsConfig := &TLSConfig{ + TLSCASecretRef: SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, + }, + ClientTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-client-tls", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-server-tls", + }, + } + + peerTlsConfig := &TLSConfig{ + TLSCASecretRef: SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "peer-url-ca-etcd", + }, + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", + }, + } + + instance := &Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: "123456", + }, + Spec: EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "role": "test", + }, + Labels: map[string]string{ + "app": "etcd-statefulset", + "role": "test", + "name": name, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd-statefulset", + "name": name, + }, + }, + Replicas: 1, + StorageClass: &storageClass, + StorageCapacity: &storageCapacity, + + Backup: BackupSpec{ + Image: &imageBR, + Port: &backupPort, + TLS: clientTlsConfig, + FullSnapshotSchedule: &snapshotSchedule, + GarbageCollectionPolicy: &garbageCollectionPolicy, + GarbageCollectionPeriod: &garbageCollectionPeriod, + DeltaSnapshotPeriod: &deltaSnapshotPeriod, + DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, + + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("23m"), + "memory": resource.MustParse("128Mi"), + }, + }, + Store: &StoreSpec{ + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + Container: &container, + Provider: &provider, + Prefix: prefix, + }, + }, + Etcd: EtcdConfig{ + Quota: "a, + Metrics: &metricLevel, + Image: &imageEtcd, + DefragmentationSchedule: &defragSchedule, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("2500m"), + "memory": resource.MustParse("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("1000Mi"), + }, + }, + ClientPort: &clientPort, + ServerPort: &serverPort, + ClientUrlTLS: clientTlsConfig, + PeerUrlTLS: peerTlsConfig, + }, + }, + } + return instance +} diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go new file mode 100644 index 000000000..64b6a2e22 --- /dev/null +++ b/api/v1alpha1/helper.go @@ -0,0 +1,128 @@ +package v1alpha1 + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" +) + +// --------------- Helper functions for Etcd resource names --------------- + +// GetPeerServiceName returns the peer service name for the Etcd cluster reachable by members within the Etcd cluster. +func GetPeerServiceName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s-peer", etcdObjMeta.Name) +} + +// GetClientServiceName returns the client service name for the Etcd cluster reachable by external clients. +func GetClientServiceName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s-client", etcdObjMeta.Name) +} + +// GetServiceAccountName returns the service account name for the Etcd. +func GetServiceAccountName(etcdObjMeta metav1.ObjectMeta) string { + return etcdObjMeta.Name +} + +// GetConfigMapName returns the name of the configmap for the Etcd. +func GetConfigMapName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("etcd-bootstrap-%s", string(etcdObjMeta.UID[:6])) +} + +// GetCompactionJobName returns the compaction job name for the Etcd. +func GetCompactionJobName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s-compactor", etcdObjMeta.Name) +} + +// GetOrdinalPodName returns the Etcd pod name based on the ordinal. +func GetOrdinalPodName(etcdObjMeta metav1.ObjectMeta, ordinal int) string { + return fmt.Sprintf("%s-%d", etcdObjMeta.Name, ordinal) +} + +// GetMemberLeaseNames returns the name of member leases for the Etcd. +func GetMemberLeaseNames(etcdObjMeta metav1.ObjectMeta, replicas int) []string { + leaseNames := make([]string, 0, replicas) + for i := 0; i < replicas; i++ { + leaseNames = append(leaseNames, fmt.Sprintf("%s-%d", etcdObjMeta.Name, i)) + } + return leaseNames +} + +// GetPodDisruptionBudgetName returns the name of the pod disruption budget for the Etcd. +func GetPodDisruptionBudgetName(etcdObjMeta metav1.ObjectMeta) string { + return etcdObjMeta.Name +} + +// GetRoleName returns the role name for the Etcd. +func GetRoleName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, etcdObjMeta.Name) +} + +// GetRoleBindingName returns the role binding name for the Etcd. +func GetRoleBindingName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s:etcd:%s", GroupVersion.Group, etcdObjMeta.Name) +} + +// GetDeltaSnapshotLeaseName returns the name of the delta snapshot lease for the Etcd. +func GetDeltaSnapshotLeaseName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s-delta-snap", etcdObjMeta.Name) +} + +// GetFullSnapshotLeaseName returns the name of the full snapshot lease for the Etcd. +func GetFullSnapshotLeaseName(etcdObjMeta metav1.ObjectMeta) string { + return fmt.Sprintf("%s-full-snap", etcdObjMeta.Name) +} + +// --------------- Miscellaneous helper functions --------------- + +// GetNamespaceName is a convenience function which creates a types.NamespacedName for an Etcd resource. +func GetNamespaceName(etcdObjMeta metav1.ObjectMeta) types.NamespacedName { + return types.NamespacedName{ + Namespace: etcdObjMeta.Namespace, + Name: etcdObjMeta.Name, + } +} + +// GetSuspendEtcdSpecReconcileAnnotationKey gets the annotation key set on an Etcd resource signalling the intent +// to suspend spec reconciliation for this Etcd resource. If no annotation is set then it will return nil. +func GetSuspendEtcdSpecReconcileAnnotationKey(etcdObjMeta metav1.ObjectMeta) *string { + if metav1.HasAnnotation(etcdObjMeta, SuspendEtcdSpecReconcileAnnotation) { + return pointer.String(SuspendEtcdSpecReconcileAnnotation) + } + if metav1.HasAnnotation(etcdObjMeta, IgnoreReconciliationAnnotation) { + return pointer.String(IgnoreReconciliationAnnotation) + } + return nil +} + +// AreManagedResourcesProtected returns false if the Etcd resource has the `druid.gardener.cloud/disable-resource-protection` annotation set, +// else returns true. +func AreManagedResourcesProtected(etcdObjMeta metav1.ObjectMeta) bool { + return !metav1.HasAnnotation(etcdObjMeta, DisableResourceProtectionAnnotation) +} + +// GetDefaultLabels returns the default labels for etcd. +func GetDefaultLabels(etcdObjMeta metav1.ObjectMeta) map[string]string { + return map[string]string{ + LabelManagedByKey: LabelManagedByValue, + LabelPartOfKey: etcdObjMeta.Name, + } +} + +// GetAsOwnerReference returns an OwnerReference object that represents the current Etcd instance. +func GetAsOwnerReference(etcdObjMeta metav1.ObjectMeta) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: GroupVersion.String(), + Kind: "Etcd", + Name: etcdObjMeta.Name, + UID: etcdObjMeta.UID, + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), + } +} + +// IsEtcdMarkedForDeletion returns true if the Etcd object is marked for deletion and false otherwise. +func IsEtcdMarkedForDeletion(etcdObjMeta metav1.ObjectMeta) bool { + return etcdObjMeta.DeletionTimestamp != nil +} diff --git a/api/v1alpha1/helper_test.go b/api/v1alpha1/helper_test.go new file mode 100644 index 000000000..afc7cd423 --- /dev/null +++ b/api/v1alpha1/helper_test.go @@ -0,0 +1,246 @@ +package v1alpha1 + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + + "k8s.io/utils/pointer" +) + +const ( + etcdName = "etcd-test" + etcdNamespace = "etcd-test-namespace" +) + +func TestGetNamespaceName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + namespaceName := GetNamespaceName(etcdObjMeta) + g.Expect(namespaceName.Namespace).To(Equal(etcdNamespace)) + g.Expect(namespaceName.Name).To(Equal(etcdName)) +} + +func TestGetPeerServiceName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + peerServiceName := GetPeerServiceName(etcdObjMeta) + g.Expect(peerServiceName).To(Equal("etcd-test-peer")) +} + +func TestGetClientServiceName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + clientServiceName := GetClientServiceName(etcdObjMeta) + g.Expect(clientServiceName).To(Equal("etcd-test-client")) +} + +func TestGetServiceAccountName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + serviceAccountName := GetServiceAccountName(etcdObjMeta) + g.Expect(serviceAccountName).To(Equal(etcdName)) +} + +func TestGetConfigMapName(t *testing.T) { + g := NewWithT(t) + uid := uuid.NewUUID() + etcdObjMeta := createEtcdObjectMetadata(uid, nil, nil, false) + configMapName := GetConfigMapName(etcdObjMeta) + g.Expect(configMapName).To(Equal("etcd-bootstrap-" + string(uid[:6]))) +} + +func TestGetCompactionJobName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + compactionJobName := GetCompactionJobName(etcdObjMeta) + g.Expect(compactionJobName).To(Equal("etcd-test-compactor")) +} + +func TestGetOrdinalPodName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + ordinalPodName := GetOrdinalPodName(etcdObjMeta, 1) + g.Expect(ordinalPodName).To(Equal("etcd-test-1")) +} + +func TestGetDeltaSnapshotLeaseName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + deltaSnapshotLeaseName := GetDeltaSnapshotLeaseName(etcdObjMeta) + g.Expect(deltaSnapshotLeaseName).To(Equal("etcd-test-delta-snap")) +} + +func TestGetFullSnapshotLeaseName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + fullSnapshotLeaseName := GetFullSnapshotLeaseName(etcdObjMeta) + g.Expect(fullSnapshotLeaseName).To(Equal("etcd-test-full-snap")) +} + +func TestGetMemberLeaseNames(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + leaseNames := GetMemberLeaseNames(etcdObjMeta, 3) + g.Expect(leaseNames).To(Equal([]string{"etcd-test-0", "etcd-test-1", "etcd-test-2"})) +} + +func TestGetPodDisruptionBudgetName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + podDisruptionBudgetName := GetPodDisruptionBudgetName(etcdObjMeta) + g.Expect(podDisruptionBudgetName).To(Equal("etcd-test")) +} + +func TestGetRoleName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + roleName := GetRoleName(etcdObjMeta) + g.Expect(roleName).To(Equal("druid.gardener.cloud:etcd:etcd-test")) +} + +func TestGetRoleBindingName(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + roleBindingName := GetRoleBindingName(etcdObjMeta) + g.Expect(roleBindingName).To(Equal("druid.gardener.cloud:etcd:etcd-test")) +} + +func TestGetSuspendEtcdSpecReconcileAnnotationKey(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + expectedAnnotationKey *string + }{ + { + name: "No annotation is set", + annotations: nil, + expectedAnnotationKey: nil, + }, + { + name: "SuspendEtcdSpecReconcileAnnotation is set", + annotations: map[string]string{SuspendEtcdSpecReconcileAnnotation: ""}, + expectedAnnotationKey: pointer.String(SuspendEtcdSpecReconcileAnnotation), + }, + { + name: "IgnoreReconciliationAnnotation is set", + annotations: map[string]string{IgnoreReconciliationAnnotation: ""}, + expectedAnnotationKey: pointer.String(IgnoreReconciliationAnnotation), + }, + { + name: "Both annotations (SuspendEtcdSpecReconcileAnnotation and IgnoreReconciliationAnnotation) are set", + annotations: map[string]string{SuspendEtcdSpecReconcileAnnotation: "", IgnoreReconciliationAnnotation: ""}, + expectedAnnotationKey: pointer.String(SuspendEtcdSpecReconcileAnnotation), + }, + } + g := NewWithT(t) + t.Parallel() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), test.annotations, nil, false) + annotationKey := GetSuspendEtcdSpecReconcileAnnotationKey(etcdObjMeta) + g.Expect(annotationKey).To(Equal(test.expectedAnnotationKey)) + }) + } +} + +func TestAreManagedResourcesProtected(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + expectedResourceProtection bool + }{ + { + name: "No DisableResourceProtectionAnnotation annotation is set", + annotations: nil, + expectedResourceProtection: true, + }, + { + name: "DisableResourceProtectionAnnotation is set", + annotations: map[string]string{DisableResourceProtectionAnnotation: ""}, + expectedResourceProtection: false, + }, + } + g := NewWithT(t) + t.Parallel() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), test.annotations, nil, false) + resourceProtection := AreManagedResourcesProtected(etcdObjMeta) + g.Expect(resourceProtection).To(Equal(test.expectedResourceProtection)) + }) + } +} + +func TestGetDefaultLabels(t *testing.T) { + g := NewWithT(t) + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, false) + defaultLabels := GetDefaultLabels(etcdObjMeta) + g.Expect(defaultLabels).To(Equal(map[string]string{ + LabelManagedByKey: LabelManagedByValue, + LabelPartOfKey: etcdName, + })) +} + +func TestGetAsOwnerReference(t *testing.T) { + g := NewWithT(t) + uid := uuid.NewUUID() + etcdObjMeta := createEtcdObjectMetadata(uid, nil, nil, false) + ownerRef := GetAsOwnerReference(etcdObjMeta) + g.Expect(ownerRef).To(Equal(metav1.OwnerReference{ + APIVersion: GroupVersion.String(), + Kind: "Etcd", + Name: etcdName, + UID: uid, + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), + })) +} + +func TestIsEtcdMarkedForDeletion(t *testing.T) { + tests := []struct { + name string + markedForDeletion bool + expectedIsMarkedForDeletion bool + }{ + { + name: "Etcd not marked for deletion", + markedForDeletion: false, + expectedIsMarkedForDeletion: false, + }, + { + name: "Etcd marked for deletion", + markedForDeletion: true, + expectedIsMarkedForDeletion: true, + }, + } + g := NewWithT(t) + t.Parallel() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etcdObjMeta := createEtcdObjectMetadata(uuid.NewUUID(), nil, nil, test.markedForDeletion) + isMarkedForDeletion := IsEtcdMarkedForDeletion(etcdObjMeta) + g.Expect(isMarkedForDeletion).To(Equal(test.expectedIsMarkedForDeletion)) + }) + } +} + +func createEtcdObjectMetadata(uid types.UID, annotations, labels map[string]string, markedForDeletion bool) metav1.ObjectMeta { + etcdObjMeta := metav1.ObjectMeta{ + Name: etcdName, + Namespace: etcdNamespace, + Labels: labels, + Annotations: annotations, + UID: uid, + } + + if markedForDeletion { + now := metav1.Now() + etcdObjMeta.DeletionTimestamp = &now + } + + return etcdObjMeta +} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/register.go similarity index 77% rename from api/v1alpha1/groupversion_info.go rename to api/v1alpha1/register.go index e627f710a..beaabf637 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/register.go @@ -13,15 +13,20 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) +// GroupName is the name of the druid API group. +const GroupName = "druid.gardener.cloud" + // nolint:gochecknoglobals var ( localSchemeBuilder = &SchemeBuilder // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "druid.gardener.cloud", Version: "v1alpha1"} + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = localSchemeBuilder.AddToScheme + // SchemeGroupVersion is group version used to register these objects. + SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} ) // Adds the list of known types to the given scheme. diff --git a/api/v1alpha1/store.go b/api/v1alpha1/store.go new file mode 100644 index 000000000..df7b188f3 --- /dev/null +++ b/api/v1alpha1/store.go @@ -0,0 +1,22 @@ +package v1alpha1 + +import corev1 "k8s.io/api/core/v1" + +// StorageProvider defines the type of object store provider for storing backups. +type StorageProvider string + +// StoreSpec defines parameters related to ObjectStore persisting backups +type StoreSpec struct { + // Container is the name of the container the backup is stored at. + // +optional + Container *string `json:"container,omitempty"` + // Prefix is the prefix used for the store. + // +required + Prefix string `json:"prefix"` + // Provider is the name of the backup provider. + // +optional + Provider *StorageProvider `json:"provider,omitempty"` + // SecretRef is the reference to the secret which used to connect to the backup store. + // +optional + SecretRef *corev1.SecretReference `json:"secretRef,omitempty"` +} diff --git a/api/v1alpha1/suite_test.go b/api/v1alpha1/suite_test.go deleted file mode 100644 index b680d367a..000000000 --- a/api/v1alpha1/suite_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs( - t, - "v1 Suite", - ) -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -}) diff --git a/api/v1alpha1/types_etcd_test.go b/api/v1alpha1/types_etcd_test.go deleted file mode 100644 index 5ca2ed01f..000000000 --- a/api/v1alpha1/types_etcd_test.go +++ /dev/null @@ -1,413 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1_test - -import ( - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - - . "github.com/gardener/etcd-druid/api/v1alpha1" -) - -// These tests are written in BDD-style using Ginkgo framework. Refer to -// http://onsi.github.io/ginkgo to learn more. - -var _ = Describe("Etcd", func() { - var ( - etcd *Etcd - ) - - BeforeEach(func() { - etcd = getEtcd("foo", "default") - }) - - Context("GetPeerServiceName", func() { - It("should return the correct peer service name", func() { - Expect(etcd.GetPeerServiceName()).To(Equal("foo-peer")) - }) - }) - - Context("GetClientServiceName", func() { - It("should return the correct client service name", func() { - Expect(etcd.GetClientServiceName()).To(Equal("foo-client")) - }) - }) - - Context("GetServiceAccountName", func() { - It("should return the correct service account name", func() { - Expect(etcd.GetServiceAccountName()).To(Equal("foo")) - }) - }) - - Context("GetConfigmapName", func() { - It("should return the correct configmap name", func() { - Expect(etcd.GetConfigMapName()).To(Equal("etcd-bootstrap-123456")) - }) - }) - - Context("GetCompactionJobName", func() { - It("should return the correct compaction job name", func() { - Expect(etcd.GetCompactionJobName()).To(Equal("foo-compactor")) - }) - }) - - Context("GetOrdinalPodName", func() { - It("should return the correct ordinal pod name", func() { - Expect(etcd.GetOrdinalPodName(0)).To(Equal("foo-0")) - }) - }) - - Context("GetDeltaSnapshotLeaseName", func() { - It("should return the correct delta snapshot lease name", func() { - Expect(etcd.GetDeltaSnapshotLeaseName()).To(Equal("foo-delta-snap")) - }) - }) - - Context("GetFullSnapshotLeaseName", func() { - It("should return the correct full snapshot lease name", func() { - Expect(etcd.GetFullSnapshotLeaseName()).To(Equal("foo-full-snap")) - }) - }) - - Context("GetDefaultLabels", func() { - It("should return the default labels for etcd", func() { - expected := map[string]string{ - LabelManagedByKey: LabelManagedByValue, - LabelPartOfKey: "foo", - } - Expect(etcd.GetDefaultLabels()).To(Equal(expected)) - }) - }) - - Context("GetAsOwnerReference", func() { - It("should return an OwnerReference object that represents the current Etcd instance", func() { - expected := metav1.OwnerReference{ - APIVersion: GroupVersion.String(), - Kind: "Etcd", - Name: "foo", - UID: "123456", - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - } - Expect(etcd.GetAsOwnerReference()).To(Equal(expected)) - }) - }) - - Context("GetRoleName", func() { - It("should return the role name for the Etcd", func() { - Expect(etcd.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) - }) - }) - - Context("GetRoleBindingName", func() { - It("should return the rolebinding name for the Etcd", func() { - Expect(etcd.GetRoleName()).To(Equal(GroupVersion.Group + ":etcd:foo")) - }) - }) - - Context("IsBackupStoreEnabled", func() { - Context("when backup is enabled", func() { - It("should return true", func() { - Expect(etcd.IsBackupStoreEnabled()).To(Equal(true)) - }) - }) - Context("when backup is not enabled", func() { - It("should return false", func() { - etcd.Spec.Backup = BackupSpec{} - Expect(etcd.IsBackupStoreEnabled()).To(Equal(false)) - }) - }) - }) - - Context("IsMarkedForDeletion", func() { - Context("when deletion timestamp is not set", func() { - It("should return false", func() { - Expect(etcd.IsMarkedForDeletion()).To(Equal(false)) - }) - }) - Context("when deletion timestamp is set", func() { - It("should return true", func() { - etcd.DeletionTimestamp = &metav1.Time{Time: time.Now()} - Expect(etcd.IsMarkedForDeletion()).To(Equal(true)) - }) - }) - }) - - Context("GetSuspendEtcdSpecReconcileAnnotationKey", func() { - ignoreReconciliationAnnotation := IgnoreReconciliationAnnotation - suspendEtcdSpecReconcileAnnotation := SuspendEtcdSpecReconcileAnnotation - Context("when etcd has only ignore-reconcile annotation set", func() { - It("should return druid.gardener.cloud/ignore-reconciliation", func() { - etcd.Annotations = map[string]string{ - IgnoreReconciliationAnnotation: "true", - } - Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&ignoreReconciliationAnnotation)) - }) - }) - Context("when etcd has only suspend-etcd-spec-reconcile annotation set", func() { - It("should return suspend-etcd-spec-reconcile annotation", func() { - etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "", - } - Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) - }) - }) - Context("when etcd has both annotations druid.gardener.cloud/suspend-etcd-spec-reconcile and druid.gardener.cloud/ignore-reconciliation set", func() { - It("should return suspend-etcd-spec-reconcile annotation", func() { - etcd.Annotations = map[string]string{ - SuspendEtcdSpecReconcileAnnotation: "", - IgnoreReconciliationAnnotation: "", - } - Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(&suspendEtcdSpecReconcileAnnotation)) - }) - }) - Context("when etcd does not have suspend-etcd-spec-reconcile or ignore-reconcile annotation set", func() { - It("should return nil string pointer", func() { - var nilString *string - Expect(etcd.GetSuspendEtcdSpecReconcileAnnotationKey()).To(Equal(nilString)) - }) - }) - }) - - Context("AreManagedResourcesProtected", func() { - Context("when etcd has annotation druid.gardener.cloud/disable-resource-protection", func() { - It("should return false", func() { - etcd.Annotations = map[string]string{ - DisableResourceProtectionAnnotation: "", - } - Expect(etcd.AreManagedResourcesProtected()).To(Equal(false)) - }) - }) - Context("when etcd does not have annotation druid.gardener.cloud/disable-resource-protection set", func() { - It("should return true", func() { - Expect(etcd.AreManagedResourcesProtected()).To(Equal(true)) - }) - }) - }) - - Context("IsReconciliationInProgress", func() { - Context("when etcd status has lastOperation, its type is Reconcile and its state is Processing", func() { - It("should return true", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeReconcile, - State: LastOperationStateProcessing, - } - Expect(etcd.IsReconciliationInProgress()).To(Equal(true)) - }) - }) - Context("when etcd status has lastOperation, its type is Reconcile and its state is Error", func() { - It("should return true", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeReconcile, - State: LastOperationStateError, - } - Expect(etcd.IsReconciliationInProgress()).To(Equal(true)) - }) - }) - Context("when etcd status has lastOperation, its type is Reconcile and its state is neither Processing or Error", func() { - It("should return false", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeReconcile, - State: LastOperationStateSucceeded, - } - Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) - }) - }) - Context("when etcd status has lastOperation, and its type is not Reconcile", func() { - It("should return false", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeCreate, - } - Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) - }) - }) - Context("when etcd status does not have lastOperation populated", func() { - It("should return false", func() { - Expect(etcd.IsReconciliationInProgress()).To(Equal(false)) - }) - }) - }) - - Context("IsDeletionInProgress", func() { - Context("when etcd status has lastOperation, its type is Delete and its state is Processing", func() { - It("should return true", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeDelete, - State: LastOperationStateProcessing, - } - Expect(etcd.IsDeletionInProgress()).To(Equal(true)) - }) - }) - Context("when etcd status has lastOperation, its type is Delete and its state is Error", func() { - It("should return true", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeDelete, - State: LastOperationStateError, - } - Expect(etcd.IsDeletionInProgress()).To(Equal(true)) - }) - }) - Context("when etcd status has lastOperation, its type is Delete and its state is neither Processing or Error", func() { - It("should return false", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeDelete, - State: LastOperationStateSucceeded, - } - Expect(etcd.IsDeletionInProgress()).To(Equal(false)) - }) - }) - Context("when etcd status has lastOperation, and its type is not Delete", func() { - It("should return false", func() { - etcd.Status.LastOperation = &LastOperation{ - Type: LastOperationTypeCreate, - } - Expect(etcd.IsDeletionInProgress()).To(Equal(false)) - }) - }) - Context("when etcd status does not have lastOperation populated", func() { - It("should return false", func() { - Expect(etcd.IsDeletionInProgress()).To(Equal(false)) - }) - }) - }) -}) - -func getEtcd(name, namespace string) *Etcd { - var ( - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - metricLevel = Basic - ) - - garbageCollectionPeriod := metav1.Duration{ - Duration: 43200 * time.Second, - } - deltaSnapshotPeriod := metav1.Duration{ - Duration: 300 * time.Second, - } - imageEtcd := "europe-docker.pkg.dev/gardener-project/public/gardener/etcd-wrapper:v0.1.0" - imageBR := "europe-docker.pkg.dev/gardener-project/public/gardener/etcdbrctl:v0.25.0" - snapshotSchedule := "0 */24 * * *" - defragSchedule := "0 */24 * * *" - container := "my-object-storage-container-name" - storageCapacity := resource.MustParse("20Gi") - deltaSnapShotMemLimit := resource.MustParse("100Mi") - quota := resource.MustParse("8Gi") - storageClass := "gardener.cloud-fast" - provider := StorageProvider("aws") - prefix := "etcd-test" - garbageCollectionPolicy := GarbageCollectionPolicy(GarbageCollectionPolicyExponential) - - clientTlsConfig := &TLSConfig{ - TLSCASecretRef: SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "client-url-ca-etcd", - }, - }, - ClientTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-client-tls", - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "client-url-etcd-server-tls", - }, - } - - peerTlsConfig := &TLSConfig{ - TLSCASecretRef: SecretReference{ - SecretReference: corev1.SecretReference{ - Name: "peer-url-ca-etcd", - }, - }, - ServerTLSSecretRef: corev1.SecretReference{ - Name: "peer-url-etcd-server-tls", - }, - } - - instance := &Etcd{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: "123456", - }, - Spec: EtcdSpec{ - Annotations: map[string]string{ - "app": "etcd-statefulset", - "role": "test", - }, - Labels: map[string]string{ - "app": "etcd-statefulset", - "role": "test", - "name": name, - }, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "name": name, - }, - }, - Replicas: 1, - StorageClass: &storageClass, - StorageCapacity: &storageCapacity, - - Backup: BackupSpec{ - Image: &imageBR, - Port: &backupPort, - TLS: clientTlsConfig, - FullSnapshotSchedule: &snapshotSchedule, - GarbageCollectionPolicy: &garbageCollectionPolicy, - GarbageCollectionPeriod: &garbageCollectionPeriod, - DeltaSnapshotPeriod: &deltaSnapshotPeriod, - DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, - - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("500m"), - "memory": resource.MustParse("2Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("23m"), - "memory": resource.MustParse("128Mi"), - }, - }, - Store: &StoreSpec{ - SecretRef: &corev1.SecretReference{ - Name: "etcd-backup", - }, - Container: &container, - Provider: &provider, - Prefix: prefix, - }, - }, - Etcd: EtcdConfig{ - Quota: "a, - Metrics: &metricLevel, - Image: &imageEtcd, - DefragmentationSchedule: &defragSchedule, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2500m"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("500m"), - "memory": resource.MustParse("1000Mi"), - }, - }, - ClientPort: &clientPort, - ServerPort: &serverPort, - ClientUrlTLS: clientTlsConfig, - PeerUrlTLS: peerTlsConfig, - }, - }, - } - return instance -} diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index a79711a35..8bb938ba7 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -193,12 +193,6 @@ spec: leadership status of corresponding etcd is checked. type: string type: object - maxBackupsLimitBasedGC: - description: MaxBackupsLimitBasedGC defines the maximum number - of Full snapshots to retain in Limit Based GarbageCollectionPolicy - All full snapshots beyond this limit will be garbage collected. - format: int32 - type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. @@ -339,8 +333,7 @@ spec: type: object type: object etcd: - description: EtcdConfig defines the configuration for the etcd cluster - to be deployed. + description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: description: SecretReference represents a Secret Reference. It @@ -1687,7 +1680,7 @@ spec: properties: autoCompactionMode: description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-etcd of backup-restore + mode or 'revision' mode for etcd and embedded-Etcd of backup-restore sidecar. enum: - periodic @@ -1695,7 +1688,7 @@ spec: type: string autoCompactionRetention: description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-etcd of backup-restore + length for etcd as well as for embedded-Etcd of backup-restore sidecar. type: string type: object @@ -1835,73 +1828,12 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: 'LastError represents the last occurred error. Deprecated: - Use LastErrors instead.' + description: LastError represents the last occurred error. type: string - lastErrors: - description: LastErrors captures errors that occurred during the last - operation. - items: - description: LastError stores details of the most recent error encountered - for a resource. - properties: - code: - description: Code is an error code that uniquely identifies - an error. - type: string - description: - description: Description is a human-readable message indicating - details of the error. - type: string - observedAt: - description: ObservedAt is the time the error was observed. - format: date-time - type: string - required: - - code - - description - - observedAt - type: object - type: array - lastOperation: - description: LastOperation indicates the last operation performed - on this resource. - properties: - description: - description: Description describes the last operation. - type: string - lastUpdateTime: - description: LastUpdateTime is the time at which the operation - was last updated. - format: date-time - type: string - runID: - description: RunID correlates an operation with a reconciliation - run. Every time an Etcd resource is reconciled (barring status - reconciliation which is periodic), a unique ID is generated - which can be used to correlate all actions done as part of a - single reconcile run. Capturing this as part of LastOperation - aids in establishing this correlation. This further helps in - also easily filtering reconcile logs as all structured logs - in a reconciliation run should have the `runID` referenced. - type: string - state: - description: State is the state of the last operation. - type: string - type: - description: Type is the type of last operation. - type: string - required: - - description - - lastUpdateTime - - runID - - state - - type - type: object members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about etcd cluster + description: EtcdMemberStatus holds information about a etcd cluster membership. properties: id: @@ -1951,7 +1883,7 @@ spec: format: int32 type: integer replicas: - description: Replicas is the replica count of the etcd cluster. + description: Replicas is the replica count of the etcd resource. format: int32 type: integer serviceName: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index a79711a35..8bb938ba7 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -193,12 +193,6 @@ spec: leadership status of corresponding etcd is checked. type: string type: object - maxBackupsLimitBasedGC: - description: MaxBackupsLimitBasedGC defines the maximum number - of Full snapshots to retain in Limit Based GarbageCollectionPolicy - All full snapshots beyond this limit will be garbage collected. - format: int32 - type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. @@ -339,8 +333,7 @@ spec: type: object type: object etcd: - description: EtcdConfig defines the configuration for the etcd cluster - to be deployed. + description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: description: SecretReference represents a Secret Reference. It @@ -1687,7 +1680,7 @@ spec: properties: autoCompactionMode: description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-etcd of backup-restore + mode or 'revision' mode for etcd and embedded-Etcd of backup-restore sidecar. enum: - periodic @@ -1695,7 +1688,7 @@ spec: type: string autoCompactionRetention: description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-etcd of backup-restore + length for etcd as well as for embedded-Etcd of backup-restore sidecar. type: string type: object @@ -1835,73 +1828,12 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: 'LastError represents the last occurred error. Deprecated: - Use LastErrors instead.' + description: LastError represents the last occurred error. type: string - lastErrors: - description: LastErrors captures errors that occurred during the last - operation. - items: - description: LastError stores details of the most recent error encountered - for a resource. - properties: - code: - description: Code is an error code that uniquely identifies - an error. - type: string - description: - description: Description is a human-readable message indicating - details of the error. - type: string - observedAt: - description: ObservedAt is the time the error was observed. - format: date-time - type: string - required: - - code - - description - - observedAt - type: object - type: array - lastOperation: - description: LastOperation indicates the last operation performed - on this resource. - properties: - description: - description: Description describes the last operation. - type: string - lastUpdateTime: - description: LastUpdateTime is the time at which the operation - was last updated. - format: date-time - type: string - runID: - description: RunID correlates an operation with a reconciliation - run. Every time an Etcd resource is reconciled (barring status - reconciliation which is periodic), a unique ID is generated - which can be used to correlate all actions done as part of a - single reconcile run. Capturing this as part of LastOperation - aids in establishing this correlation. This further helps in - also easily filtering reconcile logs as all structured logs - in a reconciliation run should have the `runID` referenced. - type: string - state: - description: State is the state of the last operation. - type: string - type: - description: Type is the type of last operation. - type: string - required: - - description - - lastUpdateTime - - runID - - state - - type - type: object members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about etcd cluster + description: EtcdMemberStatus holds information about a etcd cluster membership. properties: id: @@ -1951,7 +1883,7 @@ spec: format: int32 type: integer replicas: - description: Replicas is the replica count of the etcd cluster. + description: Replicas is the replica count of the etcd resource. format: int32 type: integer serviceName: diff --git a/internal/component/clientservice/clientservice.go b/internal/component/clientservice/clientservice.go index da9c21e13..905bd7b9c 100644 --- a/internal/component/clientservice/clientservice.go +++ b/internal/component/clientservice/clientservice.go @@ -41,9 +41,9 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the name of the existing client service for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - svcObjectKey := getObjectKey(etcd) + svcObjectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) if err := r.client.Get(ctx, svcObjectKey, objMeta); err != nil { @@ -53,9 +53,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, druiderr.WrapError(err, ErrGetClientService, "GetExistingResourceNames", - fmt.Sprintf("Error getting client service: %v for etcd: %v", svcObjectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting client service: %v for etcd: %v", svcObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -63,7 +63,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the client service for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd.ObjectMeta) svc := emptyClientService(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { buildResource(etcd, svc) @@ -73,7 +73,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncClientService, "Sync", - fmt.Sprintf("Error during create or update of client service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Error during create or update of client service: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)), ) } ctx.Logger.Info("synced", "component", "client-service", "objectKey", objectKey, "result", result) @@ -81,8 +81,8 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the client service for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { + objectKey := getObjectKey(etcdObjMeta) ctx.Logger.Info("Triggering deletion of client service", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyClientService(objectKey)); err != nil { if errors.IsNotFound(err) { @@ -103,23 +103,23 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { svc.Labels = getLabels(etcd) svc.Annotations = getAnnotations(etcd) - svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + svc.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} svc.Spec.Type = corev1.ServiceTypeClusterIP svc.Spec.SessionAffinity = corev1.ServiceAffinityNone - svc.Spec.Selector = etcd.GetDefaultLabels() + svc.Spec.Selector = druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta) svc.Spec.Ports = getPorts(etcd) } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { return client.ObjectKey{ - Name: etcd.GetClientServiceName(), - Namespace: etcd.Namespace, + Name: druidv1alpha1.GetClientServiceName(obj), + Namespace: obj.Namespace, } } func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { clientSvcLabels := map[string]string{ - druidv1alpha1.LabelAppNameKey: etcd.GetClientServiceName(), + druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetClientServiceName(etcd.ObjectMeta), druidv1alpha1.LabelComponentKey: common.ComponentNameClientService, } // Add any client service labels as defined in the etcd resource @@ -127,7 +127,7 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { if etcd.Spec.Etcd.ClientService != nil && etcd.Spec.Etcd.ClientService.Labels != nil { specClientSvcLabels = etcd.Spec.Etcd.ClientService.Labels } - return utils.MergeMaps(etcd.GetDefaultLabels(), clientSvcLabels, specClientSvcLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), clientSvcLabels, specClientSvcLabels) } func getAnnotations(etcd *druidv1alpha1.Etcd) map[string]string { diff --git a/internal/component/clientservice/clientservice_test.go b/internal/component/clientservice/clientservice_test.go index 11ab7fba3..1c13328f5 100644 --- a/internal/component/clientservice/clientservice_test.go +++ b/internal/component/clientservice/clientservice_test.go @@ -40,12 +40,12 @@ func TestGetExistingResourceNames(t *testing.T) { name: "should return the existing service name", svcExists: true, getErr: nil, - expectedServiceNames: []string{etcd.GetClientServiceName()}, + expectedServiceNames: []string{druidv1alpha1.GetClientServiceName(etcd.ObjectMeta)}, }, { name: "should return empty slice when service is not found", svcExists: false, - getErr: apierrors.NewNotFound(corev1.Resource("services"), etcd.GetClientServiceName()), + getErr: apierrors.NewNotFound(corev1.Resource("services"), druidv1alpha1.GetClientServiceName(etcd.ObjectMeta)), expectedServiceNames: []string{}, }, { @@ -65,10 +65,10 @@ func TestGetExistingResourceNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, []client.Object{newClientService(etcd)}, client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, []client.Object{newClientService(etcd)}, client.ObjectKey{Name: druidv1alpha1.GetClientServiceName(etcd.ObjectMeta), Namespace: etcd.Namespace}) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) + svcNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -113,7 +113,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := buildEtcd(tc.clientPort, tc.peerPort, tc.backupPort) - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, client.ObjectKey{Name: druidv1alpha1.GetClientServiceName(etcd.ObjectMeta), Namespace: etcd.Namespace}) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) @@ -165,7 +165,7 @@ func TestSyncWhenServiceExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // ********************* Setup ********************* - cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newClientService(existingEtcd)}, client.ObjectKey{Name: existingEtcd.GetClientServiceName(), Namespace: existingEtcd.Namespace}) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newClientService(existingEtcd)}, client.ObjectKey{Name: druidv1alpha1.GetClientServiceName(existingEtcd.ObjectMeta), Namespace: existingEtcd.Namespace}) // ********************* test sync with updated ports ********************* operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) @@ -223,11 +223,11 @@ func TestTriggerDelete(t *testing.T) { if tc.svcExists { existingObjects = append(existingObjects, newClientService(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, client.ObjectKey{Name: druidv1alpha1.GetClientServiceName(etcd.ObjectMeta), Namespace: etcd.Namespace}) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) // ********************* Test trigger delete ********************* - triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) latestClientService, getErr := getLatestClientService(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) @@ -260,17 +260,17 @@ func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Ser clientPort := utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) backupPort := utils.TypeDeref(etcd.Spec.Backup.Port, common.DefaultPortEtcdBackupRestore) peerPort := utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) - - expectedLabels := etcd.GetDefaultLabels() + etcdObjMeta := etcd.ObjectMeta + expectedLabels := druidv1alpha1.GetDefaultLabels(etcdObjMeta) var expectedAnnotations map[string]string if etcd.Spec.Etcd.ClientService != nil { expectedAnnotations = etcd.Spec.Etcd.ClientService.Annotations - expectedLabels = utils.MergeMaps(etcd.Spec.Etcd.ClientService.Labels, etcd.GetDefaultLabels()) + expectedLabels = utils.MergeMaps(etcd.Spec.Etcd.ClientService.Labels, druidv1alpha1.GetDefaultLabels(etcdObjMeta)) } g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcd.GetClientServiceName()), + "Name": Equal(druidv1alpha1.GetClientServiceName(etcd.ObjectMeta)), "Namespace": Equal(etcd.Namespace), "Annotations": testutils.MatchResourceAnnotations(expectedAnnotations), "Labels": testutils.MatchResourceLabels(expectedLabels), @@ -279,7 +279,7 @@ func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Ser "Spec": MatchFields(IgnoreExtras, Fields{ "Type": Equal(corev1.ServiceTypeClusterIP), "SessionAffinity": Equal(corev1.ServiceAffinityNone), - "Selector": Equal(etcd.GetDefaultLabels()), + "Selector": Equal(druidv1alpha1.GetDefaultLabels(etcdObjMeta)), "Ports": ConsistOf( Equal(corev1.ServicePort{ Name: "client", @@ -305,13 +305,13 @@ func matchClientService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Ser } func newClientService(etcd *druidv1alpha1.Etcd) *corev1.Service { - svc := emptyClientService(getObjectKey(etcd)) + svc := emptyClientService(getObjectKey(etcd.ObjectMeta)) buildResource(etcd, svc) return svc } func getLatestClientService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { svc := &corev1.Service{} - err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetClientServiceName(), Namespace: etcd.Namespace}, svc) + err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetClientServiceName(etcd.ObjectMeta), Namespace: etcd.Namespace}, svc) return svc, err } diff --git a/internal/component/configmap/configmap.go b/internal/component/configmap/configmap.go index 4df4d3d36..cf1928503 100644 --- a/internal/component/configmap/configmap.go +++ b/internal/component/configmap/configmap.go @@ -45,9 +45,9 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the name of the existing configmap for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - objKey := getObjectKey(etcd) + objKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) if err := r.client.Get(ctx, objKey, objMeta); err != nil { @@ -57,9 +57,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return nil, druiderr.WrapError(err, ErrGetConfigMap, "GetExistingResourceNames", - fmt.Sprintf("Error getting ConfigMap: %v for etcd: %v", objKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting ConfigMap: %v for etcd: %v", objKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -67,7 +67,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the configmap for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - cm := emptyConfigMap(getObjectKey(etcd)) + cm := emptyConfigMap(getObjectKey(etcd.ObjectMeta)) result, err := controllerutils.GetAndCreateOrMergePatch(ctx, r.client, cm, func() error { return buildResource(etcd, cm) }) @@ -75,14 +75,14 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncConfigMap, "Sync", - fmt.Sprintf("Error during create or update of configmap for etcd: %v", etcd.GetNamespaceName())) + fmt.Sprintf("Error during create or update of configmap for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } checkSum, err := computeCheckSum(cm) if err != nil { return druiderr.WrapError(err, ErrSyncConfigMap, "Sync", - fmt.Sprintf("Error when computing CheckSum for configmap for etcd: %v", etcd.GetNamespaceName())) + fmt.Sprintf("Error when computing CheckSum for configmap for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } ctx.Data[common.CheckSumKeyConfigMap] = checkSum ctx.Logger.Info("synced", "component", "configmap", "name", cm.Name, "result", result) @@ -90,8 +90,8 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the configmap for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { + objectKey := getObjectKey(etcdObjMeta) ctx.Logger.Info("Triggering deletion of ConfigMap", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyConfigMap(objectKey)); err != nil { if errors.IsNotFound(err) { @@ -115,10 +115,10 @@ func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { if err != nil { return err } - cm.Name = etcd.GetConfigMapName() + cm.Name = druidv1alpha1.GetConfigMapName(etcd.ObjectMeta) cm.Namespace = etcd.Namespace cm.Labels = getLabels(etcd) - cm.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + cm.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} cm.Data = map[string]string{etcdConfigKey: string(cfgYaml)} return nil @@ -127,15 +127,15 @@ func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { cmLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameConfigMap, - druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), + druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetConfigMapName(etcd.ObjectMeta), } - return utils.MergeMaps(etcd.GetDefaultLabels(), cmLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), cmLabels) } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { return client.ObjectKey{ - Name: etcd.GetConfigMapName(), - Namespace: etcd.Namespace, + Name: druidv1alpha1.GetConfigMapName(obj), + Namespace: obj.Namespace, } } diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index a25a9ff78..9fbea681f 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -44,9 +44,9 @@ func TestGetExistingResourceNames(t *testing.T) { expectedConfigMapNames: []string{}, }, { - name: "should return the existing congigmap name", + name: "should return the existing configmap name", cmExists: true, - expectedConfigMapNames: []string{etcd.GetConfigMapName()}, + expectedConfigMapNames: []string{druidv1alpha1.GetConfigMapName(etcd.ObjectMeta)}, }, { name: "should return error when get client get fails", @@ -68,10 +68,10 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.cmExists { existingObjects = append(existingObjects, newConfigMap(g, etcd)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - cmNames, err := operator.GetExistingResourceNames(opCtx, etcd) + cmNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -122,7 +122,7 @@ func TestSyncWhenNoConfigMapExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := buildEtcd(tc.etcdReplicas, tc.clientTLSEnabled, tc.peerTLSEnabled) - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) err := operator.Sync(opCtx, etcd) @@ -216,7 +216,7 @@ func TestSyncWhenConfigMapExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { originalEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().Build() - cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newConfigMap(g, originalEtcd)}, getObjectKey(originalEtcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newConfigMap(g, originalEtcd)}, getObjectKey(originalEtcd.ObjectMeta)) updatedEtcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithClientTLS().WithPeerTLS().Build() updatedEtcd.UID = originalEtcd.UID operator := New(cl) @@ -274,11 +274,11 @@ func TestTriggerDelete(t *testing.T) { if tc.cmExists { existingObjects = append(existingObjects, newConfigMap(g, etcd)) } - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) // ********************* Test trigger delete ********************* - triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) latestConfigMap, getErr := getLatestConfigMap(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) @@ -294,7 +294,7 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newConfigMap(g *WithT, etcd *druidv1alpha1.Etcd) *corev1.ConfigMap { - cm := emptyConfigMap(getObjectKey(etcd)) + cm := emptyConfigMap(getObjectKey(etcd.ObjectMeta)) err := buildResource(etcd, cm) g.Expect(err).ToNot(HaveOccurred()) return cm @@ -308,18 +308,19 @@ func ensureConfigMapExists(g *WithT, cl client.WithWatch, etcd *druidv1alpha1.Et func getLatestConfigMap(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.ConfigMap, error) { cm := &corev1.ConfigMap{} - err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetConfigMapName(), Namespace: etcd.Namespace}, cm) + err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetConfigMapName(etcd.ObjectMeta), Namespace: etcd.Namespace}, cm) return cm, err } func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.ConfigMap) { - expectedLabels := utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ + etcdObjMeta := etcd.ObjectMeta + expectedLabels := utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcdObjMeta), map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameConfigMap, - druidv1alpha1.LabelAppNameKey: etcd.GetConfigMapName(), + druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetConfigMapName(etcdObjMeta), }) g.Expect(actualConfigMap).To(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "ObjectMeta": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ - "Name": Equal(etcd.GetConfigMapName()), + "Name": Equal(druidv1alpha1.GetConfigMapName(etcdObjMeta)), "Namespace": Equal(etcd.Namespace), "Labels": testutils.MatchResourceLabels(expectedLabels), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), @@ -353,7 +354,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -371,6 +372,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { + peerSvcName := druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta) if etcd.Spec.Etcd.PeerUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ @@ -381,12 +383,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "auto-tls": Equal(false), }), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", peerSvcName, etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", etcd.GetPeerServiceName(), etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", peerSvcName, etcd.Namespace, strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } diff --git a/internal/component/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go index adb810ccc..1e5871eb1 100644 --- a/internal/component/configmap/etcdconfig.go +++ b/internal/component/configmap/etcdconfig.go @@ -69,7 +69,7 @@ type securityConfig struct { func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { clientScheme, clientSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.ClientUrlTLS, common.VolumeMountPathEtcdCA, common.VolumeMountPathEtcdServerTLS) peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, common.VolumeMountPathEtcdPeerCA, common.VolumeMountPathEtcdPeerServerTLS) - + peerSvcName := druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta) cfg := &etcdConfig{ Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), DataDir: defaultDataDir, @@ -84,8 +84,8 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionRetention: utils.TypeDeref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), - AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), - AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, etcd.GetPeerServiceName(), etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), + AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, peerSvcName, etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), + AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, peerSvcName, etcd.Namespace, utils.TypeDeref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -120,11 +120,11 @@ func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, caPath, serv } func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { - domainName := fmt.Sprintf("%s.%s.%s", etcd.GetPeerServiceName(), etcd.Namespace, "svc") + domainName := fmt.Sprintf("%s.%s.%s", druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), etcd.Namespace, "svc") serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))) builder := strings.Builder{} for i := 0; i < int(etcd.Spec.Replicas); i++ { - podName := etcd.GetOrdinalPodName(i) + podName := druidv1alpha1.GetOrdinalPodName(etcd.ObjectMeta, i) builder.WriteString(fmt.Sprintf("%s=%s://%s.%s:%s,", podName, peerScheme, podName, domainName, serverPort)) } return strings.Trim(builder.String(), ",") diff --git a/internal/component/memberlease/memberlease.go b/internal/component/memberlease/memberlease.go index 9738c2264..8ff4caa89 100644 --- a/internal/component/memberlease/memberlease.go +++ b/internal/component/memberlease/memberlease.go @@ -40,23 +40,23 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the names of the existing member leases for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) objMetaList := &metav1.PartialObjectMetadataList{} objMetaList.SetGroupVersionKind(coordinationv1.SchemeGroupVersion.WithKind("Lease")) if err := r.client.List(ctx, objMetaList, - client.InNamespace(etcd.Namespace), - client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd)), + client.InNamespace(etcdObjMeta.Namespace), + client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcdObjMeta)), ); err != nil { return resourceNames, druiderr.WrapError(err, ErrListMemberLease, "GetExistingResourceNames", - fmt.Sprintf("Error listing member leases for etcd: %v", etcd.GetNamespaceName())) + fmt.Sprintf("Error listing member leases for etcd: %v", druidv1alpha1.GetNamespaceName(etcdObjMeta))) } for _, lease := range objMetaList.Items { - if metav1.IsControlledBy(&lease, etcd) { + if metav1.IsControlledBy(&lease, &etcdObjMeta) { resourceNames = append(resourceNames, lease.Name) } } @@ -96,23 +96,23 @@ func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1 return druiderr.WrapError(err, ErrSyncMemberLease, "Sync", - fmt.Sprintf("Error syncing member lease: %v for etcd: %v", objKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error syncing member lease: %v for etcd: %v", objKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } ctx.Logger.Info("triggered create or update of member lease", "objectKey", objKey, "operationResult", opResult) return nil } // TriggerDelete deletes the member leases for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { ctx.Logger.Info("Triggering deletion of member leases") if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, - client.InNamespace(etcd.Namespace), - client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcd))); err != nil { + client.InNamespace(etcdObjMeta.Namespace), + client.MatchingLabels(getSelectorLabelsForAllMemberLeases(etcdObjMeta))); err != nil { return druiderr.WrapError(err, ErrDeleteMemberLease, "TriggerDelete", - fmt.Sprintf("Failed to delete member leases for etcd: %v", etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete member leases for etcd: %v", druidv1alpha1.GetNamespaceName(etcdObjMeta))) } ctx.Logger.Info("deleted", "component", "member-leases") return nil @@ -120,11 +120,11 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { lease.Labels = getLabels(etcd, lease.Name) - lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + lease.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} } func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { - leaseNames := etcd.GetMemberLeaseNames() + leaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas)) objectKeys := make([]client.ObjectKey, 0, len(leaseNames)) for _, leaseName := range leaseNames { objectKeys = append(objectKeys, client.ObjectKey{Name: leaseName, Namespace: etcd.Namespace}) @@ -132,11 +132,11 @@ func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { return objectKeys } -func getSelectorLabelsForAllMemberLeases(etcd *druidv1alpha1.Etcd) map[string]string { +func getSelectorLabelsForAllMemberLeases(etcdObjMeta metav1.ObjectMeta) map[string]string { leaseMatchingLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, } - return utils.MergeMaps(etcd.GetDefaultLabels(), leaseMatchingLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcdObjMeta), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { @@ -144,7 +144,7 @@ func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, druidv1alpha1.LabelAppNameKey: leaseName, } - return utils.MergeMaps(leaseLabels, etcd.GetDefaultLabels()) + return utils.MergeMaps(leaseLabels, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)) } func emptyMemberLease(objectKey client.ObjectKey) *coordinationv1.Lease { diff --git a/internal/component/memberlease/memberlease_test.go b/internal/component/memberlease/memberlease_test.go index 37d49c18a..43a1a2468 100644 --- a/internal/component/memberlease/memberlease_test.go +++ b/internal/component/memberlease/memberlease_test.go @@ -87,15 +87,15 @@ func TestGetExistingResourceNames(t *testing.T) { existingObjects = append(existingObjects, lease) } } - cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd), existingObjects...) + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd.ObjectMeta), existingObjects...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - memberLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) + memberLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) - expectedLeaseNames := etcd.GetMemberLeaseNames()[:tc.numExistingLeases] + expectedLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas))[:tc.numExistingLeases] g.Expect(memberLeaseNames).To(Equal(expectedLeaseNames)) } }) @@ -249,11 +249,11 @@ func TestTriggerDelete(t *testing.T) { for _, nonTargetLeaseName := range nonTargetLeaseNames { existingObjects = append(existingObjects, testutils.CreateLease(nonTargetLeaseName, nonTargetEtcd.Namespace, nonTargetEtcd.Name, nonTargetEtcd.UID, common.ComponentNameMemberLease)) } - cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd), existingObjects...) + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, etcd.Namespace, getSelectorLabelsForAllMemberLeases(etcd.ObjectMeta), existingObjects...) // ***************** Setup component operator and test ***************** operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.TriggerDelete(opCtx, etcd) + err := operator.TriggerDelete(opCtx, etcd.ObjectMeta) memberLeasesPostDelete := getLatestMemberLeases(g, cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) @@ -275,7 +275,7 @@ func getLatestMemberLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd) etcd, utils.MergeMaps(map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, - }, etcd.GetDefaultLabels())) + }, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta))) } func doGetLatestLeases(g *WithT, cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) []coordinationv1.Lease { @@ -309,7 +309,7 @@ func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 if numLeases > int(etcd.Spec.Replicas) { return nil, errors.New("number of requested leases is greater than the etcd replicas") } - memberLeaseNames := etcd.GetMemberLeaseNames() + memberLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas)) leases := make([]*coordinationv1.Lease, 0, numLeases) for i := 0; i < numLeases; i++ { lease := &coordinationv1.Lease{ @@ -317,7 +317,7 @@ func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 Name: memberLeaseNames[i], Namespace: etcd.Namespace, Labels: getLabels(etcd, memberLeaseNames[i]), - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + OwnerReferences: []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)}, }, } leases = append(leases, lease) diff --git a/internal/component/peerservice/peerservice.go b/internal/component/peerservice/peerservice.go index 293425296..6d9f62ef1 100644 --- a/internal/component/peerservice/peerservice.go +++ b/internal/component/peerservice/peerservice.go @@ -41,9 +41,9 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the name of the existing peer service for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - svcObjectKey := getObjectKey(etcd) + svcObjectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) if err := r.client.Get(ctx, svcObjectKey, objMeta); err != nil { @@ -53,9 +53,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, druiderr.WrapError(err, ErrGetPeerService, "GetExistingResourceNames", - fmt.Sprintf("Error getting peer service: %s for etcd: %v", svcObjectKey.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting peer service: %s for etcd: %v", svcObjectKey.Name, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -63,7 +63,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the peer service for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd.ObjectMeta) svc := emptyPeerService(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, svc, func() error { buildResource(etcd, svc) @@ -73,7 +73,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncPeerService, "Sync", - fmt.Sprintf("Error during create or update of peer service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Error during create or update of peer service: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)), ) } ctx.Logger.Info("synced", "component", "peer-service", "objectKey", objectKey, "result", result) @@ -81,8 +81,8 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the peer service for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { + objectKey := getObjectKey(etcdObjMeta) ctx.Logger.Info("Triggering deletion of peer service") if err := r.client.Delete(ctx, emptyPeerService(objectKey)); err != nil { if errors.IsNotFound(err) { @@ -93,7 +93,7 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp err, ErrDeletePeerService, "TriggerDelete", - fmt.Sprintf("Failed to delete peer service: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Failed to delete peer service: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), ) } ctx.Logger.Info("deleted", "component", "peer-service", "objectKey", objectKey) @@ -102,11 +102,11 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { svc.Labels = getLabels(etcd) - svc.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + svc.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} svc.Spec.Type = corev1.ServiceTypeClusterIP svc.Spec.ClusterIP = corev1.ClusterIPNone svc.Spec.SessionAffinity = corev1.ServiceAffinityNone - svc.Spec.Selector = etcd.GetDefaultLabels() + svc.Spec.Selector = druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta) svc.Spec.PublishNotReadyAddresses = true svc.Spec.Ports = getPorts(etcd) } @@ -114,13 +114,13 @@ func buildResource(etcd *druidv1alpha1.Etcd, svc *corev1.Service) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { svcLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNamePeerService, - druidv1alpha1.LabelAppNameKey: etcd.GetPeerServiceName(), + druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), } - return utils.MergeMaps(etcd.GetDefaultLabels(), svcLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), svcLabels) } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { - return client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace} +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { + return client.ObjectKey{Name: druidv1alpha1.GetPeerServiceName(obj), Namespace: obj.Namespace} } func emptyPeerService(objectKey client.ObjectKey) *corev1.Service { diff --git a/internal/component/peerservice/peerservice_test.go b/internal/component/peerservice/peerservice_test.go index 571891a0c..1fa182a1d 100644 --- a/internal/component/peerservice/peerservice_test.go +++ b/internal/component/peerservice/peerservice_test.go @@ -39,7 +39,7 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return the existing service name", svcExists: true, - expectedServiceNames: []string{etcd.GetPeerServiceName()}, + expectedServiceNames: []string{druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)}, }, { name: "should return empty slice when service is not found", @@ -68,10 +68,10 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.svcExists { existingObjects = append(existingObjects, newPeerService(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - svcNames, err := operator.GetExistingResourceNames(opCtx, etcd) + svcNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -116,7 +116,7 @@ func TestSyncWhenNoServiceExists(t *testing.T) { etcdBuilder.WithEtcdServerPort(tc.createWithPort) } etcd := etcdBuilder.Build() - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -162,7 +162,7 @@ func TestSyncWhenServiceExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { existingEtcd := etcdBuilder.Build() - cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newPeerService(existingEtcd)}, getObjectKey(existingEtcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newPeerService(existingEtcd)}, getObjectKey(existingEtcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithEtcdServerPort(tc.updateWithPort).Build() @@ -214,10 +214,10 @@ func TestPeerServiceTriggerDelete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, []client.Object{newPeerService(etcd)}, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, []client.Object{newPeerService(etcd)}, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - syncErr := operator.TriggerDelete(opCtx, etcd) + syncErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) _, getErr := getLatestPeerService(cl, etcd) if tc.expectError != nil { testutils.CheckDruidError(g, tc.expectError, syncErr) @@ -233,25 +233,26 @@ func TestPeerServiceTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newPeerService(etcd *druidv1alpha1.Etcd) *corev1.Service { - svc := emptyPeerService(getObjectKey(etcd)) + svc := emptyPeerService(getObjectKey(etcd.ObjectMeta)) buildResource(etcd, svc) return svc } func matchPeerService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Service) { peerPort := utils.TypeDeref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + etcdObjMeta := etcd.ObjectMeta g.Expect(actualSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcd.GetPeerServiceName()), + "Name": Equal(druidv1alpha1.GetPeerServiceName(etcdObjMeta)), "Namespace": Equal(etcd.Namespace), - "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "Labels": testutils.MatchResourceLabels(druidv1alpha1.GetDefaultLabels(etcdObjMeta)), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), "Spec": MatchFields(IgnoreExtras, Fields{ "Type": Equal(corev1.ServiceTypeClusterIP), "ClusterIP": Equal(corev1.ClusterIPNone), "SessionAffinity": Equal(corev1.ServiceAffinityNone), - "Selector": Equal(etcd.GetDefaultLabels()), + "Selector": Equal(druidv1alpha1.GetDefaultLabels(etcdObjMeta)), "Ports": ConsistOf( Equal(corev1.ServicePort{ Name: "peer", @@ -266,6 +267,6 @@ func matchPeerService(g *WithT, etcd *druidv1alpha1.Etcd, actualSvc corev1.Servi func getLatestPeerService(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.Service, error) { svc := &corev1.Service{} - err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetPeerServiceName(), Namespace: etcd.Namespace}, svc) + err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), Namespace: etcd.Namespace}, svc) return svc, err } diff --git a/internal/component/poddistruptionbudget/poddisruptionbudget.go b/internal/component/poddistruptionbudget/poddisruptionbudget.go index 8f09f7aa5..c0c6dd7a8 100644 --- a/internal/component/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/component/poddistruptionbudget/poddisruptionbudget.go @@ -41,9 +41,9 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the name of the existing pod disruption budget for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget")) if err := r.client.Get(ctx, objectKey, objMeta); err != nil { @@ -53,9 +53,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, druiderr.WrapError(err, ErrGetPodDisruptionBudget, "GetExistingResourceNames", - fmt.Sprintf("Error getting PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting PDB: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -63,7 +63,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the pod disruption budget for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd.ObjectMeta) pdb := emptyPodDisruptionBudget(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, pdb, func() error { buildResource(etcd, pdb) @@ -73,7 +73,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncPodDisruptionBudget, "Sync", - fmt.Sprintf("Error during create or update of PDB: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Error during create or update of PDB: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)), ) } ctx.Logger.Info("synced", "component", "pod-disruption-budget", "objectKey", objectKey, "result", result) @@ -81,14 +81,14 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the pod disruption budget for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { ctx.Logger.Info("Triggering deletion of PDB") - pdbObjectKey := getObjectKey(etcd) + pdbObjectKey := getObjectKey(etcdObjMeta) if err := client.IgnoreNotFound(r.client.Delete(ctx, emptyPodDisruptionBudget(pdbObjectKey))); err != nil { return druiderr.WrapError(err, ErrDeletePodDisruptionBudget, "TriggerDelete", - fmt.Sprintf("Failed to delete PDB: %v for etcd: %v", pdbObjectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete PDB: %v for etcd: %v", pdbObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } ctx.Logger.Info("deleted", "component", "pod-disruption-budget", "objectKey", pdbObjectKey) return nil @@ -96,13 +96,13 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp func buildResource(etcd *druidv1alpha1.Etcd, pdb *policyv1.PodDisruptionBudget) { pdb.Labels = getLabels(etcd) - pdb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + pdb.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} pdb.Spec.MinAvailable = &intstr.IntOrString{ IntVal: computePDBMinAvailable(int(etcd.Spec.Replicas)), Type: intstr.Int, } pdb.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: etcd.GetDefaultLabels(), + MatchLabels: druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), } } @@ -111,11 +111,11 @@ func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { druidv1alpha1.LabelComponentKey: common.ComponentNamePodDisruptionBudget, druidv1alpha1.LabelAppNameKey: etcd.Name, } - return utils.MergeMaps(etcd.GetDefaultLabels(), pdbLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), pdbLabels) } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { - return client.ObjectKey{Name: etcd.Name, Namespace: etcd.Namespace} +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { + return client.ObjectKey{Name: druidv1alpha1.GetPodDisruptionBudgetName(obj), Namespace: obj.Namespace} } func emptyPodDisruptionBudget(objectKey client.ObjectKey) *policyv1.PodDisruptionBudget { diff --git a/internal/component/poddistruptionbudget/poddisruptionbudget_test.go b/internal/component/poddistruptionbudget/poddisruptionbudget_test.go index 2abd1c425..fa24296de 100644 --- a/internal/component/poddistruptionbudget/poddisruptionbudget_test.go +++ b/internal/component/poddistruptionbudget/poddisruptionbudget_test.go @@ -42,7 +42,7 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return empty slice when PDB is not found", pdbExists: false, - getErr: apierrors.NewNotFound(corev1.Resource("services"), etcd.GetClientServiceName()), + getErr: apierrors.NewNotFound(corev1.Resource("poddisruptionbudgets"), druidv1alpha1.GetPodDisruptionBudgetName(etcd.ObjectMeta)), expectedPDBNames: []string{}, }, { @@ -66,10 +66,10 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.pdbExists { existingObjects = append(existingObjects, newPodDisruptionBudget(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - pdbNames, err := operator.GetExistingResourceNames(opCtx, etcd) + pdbNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -116,7 +116,7 @@ func TestSyncWhenNoPDBExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := etcdBuilder.WithReplicas(tc.etcdReplicas).Build() - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -169,7 +169,7 @@ func TestSyncWhenPDBExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { existingEtcd := etcdBuilder.WithReplicas(tc.originalEtcdReplicas).Build() - cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newPodDisruptionBudget(existingEtcd)}, getObjectKey(existingEtcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, tc.patchErr, nil, []client.Object{newPodDisruptionBudget(existingEtcd)}, getObjectKey(existingEtcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) updatedEtcd := etcdBuilder.WithReplicas(tc.updatedEtcdReplicas).Build() @@ -223,10 +223,10 @@ func TestTriggerDelete(t *testing.T) { if tc.pdbExists { existingObjects = append(existingObjects, newPodDisruptionBudget(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - syncErr := operator.TriggerDelete(opCtx, etcd) + syncErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) _, getErr := getLatestPodDisruptionBudget(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, syncErr) @@ -242,7 +242,7 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newPodDisruptionBudget(etcd *druidv1alpha1.Etcd) *policyv1.PodDisruptionBudget { - pdb := emptyPodDisruptionBudget(getObjectKey(etcd)) + pdb := emptyPodDisruptionBudget(getObjectKey(etcd.ObjectMeta)) buildResource(etcd, pdb) return pdb } @@ -256,13 +256,13 @@ func getLatestPodDisruptionBudget(cl client.Client, etcd *druidv1alpha1.Etcd) (* func matchPodDisruptionBudget(g *WithT, etcd *druidv1alpha1.Etcd, actualPDB policyv1.PodDisruptionBudget, expectedPDBMinAvailable int32) { g.Expect(actualPDB).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcd.Name), + "Name": Equal(druidv1alpha1.GetPodDisruptionBudgetName(etcd.ObjectMeta)), "Namespace": Equal(etcd.Namespace), - "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "Labels": testutils.MatchResourceLabels(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), "Spec": MatchFields(IgnoreExtras, Fields{ - "Selector": testutils.MatchSpecLabelSelector(etcd.GetDefaultLabels()), + "Selector": testutils.MatchSpecLabelSelector(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)), "MinAvailable": PointTo(MatchFields(IgnoreExtras, Fields{ "Type": Equal(intstr.Int), "IntVal": Equal(expectedPDBMinAvailable), diff --git a/internal/component/role/role.go b/internal/component/role/role.go index 216d601ab..ccd231cc1 100644 --- a/internal/component/role/role.go +++ b/internal/component/role/role.go @@ -41,9 +41,9 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the name of the existing role for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) if err := r.client.Get(ctx, objectKey, objMeta); err != nil { @@ -53,9 +53,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, druiderr.WrapError(err, ErrGetRole, "GetExistingResourceNames", - fmt.Sprintf("Error getting role: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting role: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -63,7 +63,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the role for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd.ObjectMeta) role := emptyRole(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, role, func() error { buildResource(etcd, role) @@ -73,7 +73,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncRole, "Sync", - fmt.Sprintf("Error during create or update of role %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Error during create or update of role %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)), ) } ctx.Logger.Info("synced", "component", "role", "objectKey", objectKey, "result", result) @@ -81,8 +81,8 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the role for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { + objectKey := getObjectKey(etcdObjMeta) ctx.Logger.Info("Triggering deletion of role", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyRole(objectKey)); err != nil { if errors.IsNotFound(err) { @@ -92,15 +92,15 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return druiderr.WrapError(err, ErrDeleteRole, "TriggerDelete", - fmt.Sprintf("Failed to delete role: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Failed to delete role: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), ) } ctx.Logger.Info("deleted", "component", "role", "objectKey", objectKey) return nil } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { - return client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace} +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { + return client.ObjectKey{Name: druidv1alpha1.GetRoleName(obj), Namespace: obj.Namespace} } func emptyRole(objectKey client.ObjectKey) *rbacv1.Role { @@ -114,7 +114,7 @@ func emptyRole(objectKey client.ObjectKey) *rbacv1.Role { func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { role.Labels = getLabels(etcd) - role.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + role.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} role.Rules = []rbacv1.PolicyRule{ { APIGroups: []string{"coordination.k8s.io"}, @@ -137,7 +137,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, role *rbacv1.Role) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameRole, - druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleName(), ":", "-"), // role name contains `:` which is not an allowed character as a label value. + druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(druidv1alpha1.GetRoleName(etcd.ObjectMeta), ":", "-"), // role name contains `:` which is not an allowed character as a label value. } - return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), roleLabels) } diff --git a/internal/component/role/role_test.go b/internal/component/role/role_test.go index 694d6940e..4bbd01f27 100644 --- a/internal/component/role/role_test.go +++ b/internal/component/role/role_test.go @@ -35,12 +35,12 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return the existing role name", roleExists: true, - expectedRoleNames: []string{etcd.GetRoleName()}, + expectedRoleNames: []string{druidv1alpha1.GetRoleName(etcd.ObjectMeta)}, }, { name: "should return empty slice when role is not found", roleExists: false, - getErr: apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleName()), + getErr: apierrors.NewNotFound(corev1.Resource("roles"), druidv1alpha1.GetRoleName(etcd.ObjectMeta)), expectedRoleNames: []string{}, }, { @@ -64,10 +64,10 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.roleExists { existingObjects = append(existingObjects, newRole(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - roleNames, err := operator.GetExistingResourceNames(opCtx, etcd) + roleNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -105,7 +105,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -161,10 +161,10 @@ func TestTriggerDelete(t *testing.T) { if tc.roleExists { existingObjects = append(existingObjects, newRole(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - deleteErr := operator.TriggerDelete(opCtx, etcd) + deleteErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) latestRole, getErr := getLatestRole(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, deleteErr) @@ -181,23 +181,23 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newRole(etcd *druidv1alpha1.Etcd) *rbacv1.Role { - role := emptyRole(getObjectKey(etcd)) + role := emptyRole(getObjectKey(etcd.ObjectMeta)) buildResource(etcd, role) return role } func getLatestRole(cl client.Client, etcd *druidv1alpha1.Etcd) (*rbacv1.Role, error) { role := &rbacv1.Role{} - err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleName(), Namespace: etcd.Namespace}, role) + err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetRoleName(etcd.ObjectMeta), Namespace: etcd.Namespace}, role) return role, err } func matchRole(g *WithT, etcd *druidv1alpha1.Etcd, actualRole rbacv1.Role) { g.Expect(actualRole).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcd.GetRoleName()), + "Name": Equal(druidv1alpha1.GetRoleName(etcd.ObjectMeta)), "Namespace": Equal(etcd.Namespace), - "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "Labels": testutils.MatchResourceLabels(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), "Rules": ConsistOf( diff --git a/internal/component/rolebinding/rolebinding.go b/internal/component/rolebinding/rolebinding.go index 472e5e043..9be850052 100644 --- a/internal/component/rolebinding/rolebinding.go +++ b/internal/component/rolebinding/rolebinding.go @@ -41,9 +41,9 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the name of the existing role binding for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) if err := r.client.Get(ctx, objectKey, objMeta); err != nil { @@ -53,9 +53,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, druiderr.WrapError(err, ErrGetRoleBinding, "GetExistingResourceNames", - fmt.Sprintf("Error getting role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting role-binding: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -63,7 +63,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the role binding for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd.ObjectMeta) rb := emptyRoleBinding(objectKey) result, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, rb, func() error { buildResource(etcd, rb) @@ -73,7 +73,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncRoleBinding, "Sync", - fmt.Sprintf("Error during create or update of role-binding %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Error during create or update of role-binding %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)), ) } ctx.Logger.Info("synced", "component", "role", "objectKey", objectKey, "result", result) @@ -81,8 +81,8 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the role binding for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { + objectKey := getObjectKey(etcdObjMeta) ctx.Logger.Info("Triggering deletion of role", "objectKey", objectKey) if err := r.client.Delete(ctx, emptyRoleBinding(objectKey)); err != nil { if errors.IsNotFound(err) { @@ -92,15 +92,15 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return druiderr.WrapError(err, ErrDeleteRoleBinding, "TriggerDelete", - fmt.Sprintf("Failed to delete role-binding: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Failed to delete role-binding: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), ) } ctx.Logger.Info("deleted", "component", "role-binding", "objectKey", objectKey) return nil } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { - return client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace} +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { + return client.ObjectKey{Name: druidv1alpha1.GetRoleBindingName(obj), Namespace: obj.Namespace} } func emptyRoleBinding(objKey client.ObjectKey) *rbacv1.RoleBinding { @@ -114,16 +114,16 @@ func emptyRoleBinding(objKey client.ObjectKey) *rbacv1.RoleBinding { func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { rb.Labels = getLabels(etcd) - rb.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + rb.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} rb.RoleRef = rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "Role", - Name: etcd.GetRoleName(), + Name: druidv1alpha1.GetRoleName(etcd.ObjectMeta), } rb.Subjects = []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: etcd.GetServiceAccountName(), + Name: druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), Namespace: etcd.Namespace, }, } @@ -132,7 +132,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, rb *rbacv1.RoleBinding) { func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameRoleBinding, - druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(etcd.GetRoleBindingName(), ":", "-"), // role-binding name contains `:` which is not an allowed character as a label value. + druidv1alpha1.LabelAppNameKey: strings.ReplaceAll(druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta), ":", "-"), // role-binding name contains `:` which is not an allowed character as a label value. } - return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), roleLabels) } diff --git a/internal/component/rolebinding/rolebinding_test.go b/internal/component/rolebinding/rolebinding_test.go index 13e8fdd0c..57eefc5ca 100644 --- a/internal/component/rolebinding/rolebinding_test.go +++ b/internal/component/rolebinding/rolebinding_test.go @@ -36,12 +36,12 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return the existing role binding name", roleBindingExists: true, - expectedRoleBindingNames: []string{etcd.GetRoleBindingName()}, + expectedRoleBindingNames: []string{druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta)}, }, { name: "should return empty slice when role binding is not found", roleBindingExists: false, - getErr: apierrors.NewNotFound(corev1.Resource("roles"), etcd.GetRoleBindingName()), + getErr: apierrors.NewNotFound(corev1.Resource("roles"), druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta)), expectedRoleBindingNames: []string{}, }, { @@ -65,10 +65,10 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.roleBindingExists { existingObjects = append(existingObjects, newRoleBinding(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - roleBindingNames, err := operator.GetExistingResourceNames(opCtx, etcd) + roleBindingNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -106,7 +106,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -162,16 +162,16 @@ func TestTriggerDelete(t *testing.T) { if tc.roleBindingExists { existingObjects = append(existingObjects, newRoleBinding(etcd)) } - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - err := operator.TriggerDelete(opCtx, etcd) + err := operator.TriggerDelete(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).NotTo(HaveOccurred()) existingRoleBinding := rbacv1.RoleBinding{} - err = cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace}, &existingRoleBinding) + err = cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta), Namespace: etcd.Namespace}, &existingRoleBinding) g.Expect(err).To(HaveOccurred()) g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) } @@ -182,34 +182,34 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newRoleBinding(etcd *druidv1alpha1.Etcd) *rbacv1.RoleBinding { - rb := emptyRoleBinding(getObjectKey(etcd)) + rb := emptyRoleBinding(getObjectKey(etcd.ObjectMeta)) buildResource(etcd, rb) return rb } func getLatestRoleBinding(cl client.Client, etcd *druidv1alpha1.Etcd) (*rbacv1.RoleBinding, error) { rb := &rbacv1.RoleBinding{} - err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetRoleBindingName(), Namespace: etcd.Namespace}, rb) + err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta), Namespace: etcd.Namespace}, rb) return rb, err } func matchRoleBinding(g *WithT, etcd *druidv1alpha1.Etcd, actualRoleBinding rbacv1.RoleBinding) { g.Expect(actualRoleBinding).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcd.GetRoleBindingName()), + "Name": Equal(druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta)), "Namespace": Equal(etcd.Namespace), - "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "Labels": testutils.MatchResourceLabels(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), "RoleRef": Equal(rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "Role", - Name: etcd.GetRoleName(), + Name: druidv1alpha1.GetRoleName(etcd.ObjectMeta), }), "Subjects": ConsistOf( rbacv1.Subject{ Kind: "ServiceAccount", - Name: etcd.GetServiceAccountName(), + Name: druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), Namespace: etcd.Namespace, }, ), diff --git a/internal/component/serviceaccount/serviceaccount.go b/internal/component/serviceaccount/serviceaccount.go index 638805541..dbbf61281 100644 --- a/internal/component/serviceaccount/serviceaccount.go +++ b/internal/component/serviceaccount/serviceaccount.go @@ -43,9 +43,9 @@ func New(client client.Client, disableAutomount bool) component.Operator { } // GetExistingResourceNames returns the name of the existing service account for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) if err := r.client.Get(ctx, objectKey, objMeta); err != nil { @@ -55,9 +55,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return resourceNames, druiderr.WrapError(err, ErrGetServiceAccount, "GetExistingResourceNames", - fmt.Sprintf("Error getting service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting service account: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -65,7 +65,7 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd // Sync creates or updates the service account for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcd.ObjectMeta) sa := emptyServiceAccount(objectKey) opResult, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, r.client, sa, func() error { buildResource(etcd, sa, !r.disableAutoMount) @@ -75,7 +75,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncServiceAccount, "Sync", - fmt.Sprintf("Error during create or update of service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName()), + fmt.Sprintf("Error during create or update of service account: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)), ) } ctx.Logger.Info("synced", "component", "service-account", "objectKey", objectKey, "result", opResult) @@ -83,9 +83,9 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the service account for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { ctx.Logger.Info("Triggering deletion of service account") - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcdObjMeta) if err := r.client.Delete(ctx, emptyServiceAccount(objectKey)); err != nil { if errors.IsNotFound(err) { ctx.Logger.Info("No ServiceAccount found, Deletion is a No-Op", "objectKey", objectKey) @@ -94,7 +94,7 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return druiderr.WrapError(err, ErrDeleteServiceAccount, "TriggerDelete", - fmt.Sprintf("Failed to delete service account: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete service account: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } ctx.Logger.Info("deleted", "component", "service-account", "objectKey", objectKey) return nil @@ -102,20 +102,20 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp func buildResource(etcd *druidv1alpha1.Etcd, sa *corev1.ServiceAccount, autoMountServiceAccountToken bool) { sa.Labels = getLabels(etcd) - sa.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + sa.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} sa.AutomountServiceAccountToken = pointer.Bool(autoMountServiceAccountToken) } func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { roleLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameServiceAccount, - druidv1alpha1.LabelAppNameKey: etcd.GetServiceAccountName(), + druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), } - return utils.MergeMaps(etcd.GetDefaultLabels(), roleLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), roleLabels) } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { - return client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace} +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { + return client.ObjectKey{Name: druidv1alpha1.GetServiceAccountName(obj), Namespace: obj.Namespace} } func emptyServiceAccount(objectKey client.ObjectKey) *corev1.ServiceAccount { diff --git a/internal/component/serviceaccount/serviceaccount_test.go b/internal/component/serviceaccount/serviceaccount_test.go index f56896f53..13e0a48ae 100644 --- a/internal/component/serviceaccount/serviceaccount_test.go +++ b/internal/component/serviceaccount/serviceaccount_test.go @@ -35,13 +35,13 @@ func TestGetExistingResourceNames(t *testing.T) { { name: "should return empty slice, when no service account exists", saExists: false, - getErr: apierrors.NewNotFound(corev1.Resource("serviceaccounts"), etcd.GetServiceAccountName()), + getErr: apierrors.NewNotFound(corev1.Resource("serviceaccounts"), druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta)), expectedSANames: []string{}, }, { name: "should return existing service account name", saExists: true, - expectedSANames: []string{etcd.GetServiceAccountName()}, + expectedSANames: []string{druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta)}, }, { name: "should return err when client get fails", @@ -63,10 +63,10 @@ func TestGetExistingResourceNames(t *testing.T) { if tc.saExists { existingObjects = append(existingObjects, newServiceAccount(etcd, false)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl, true) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - saNames, err := operator.GetExistingResourceNames(opCtx, etcd) + saNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -114,7 +114,7 @@ func TestSync(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).Build() - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, nil, getObjectKey(etcd.ObjectMeta)) operator := New(cl, tc.disableAutoMount) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, etcd) @@ -168,10 +168,10 @@ func TestTriggerDelete(t *testing.T) { if tc.saExists { existingObjects = append(existingObjects, newServiceAccount(etcd, false)) } - cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, nil, nil, tc.deleteErr, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl, false) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) latestSA, getErr := getLatestServiceAccount(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) @@ -187,23 +187,23 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newServiceAccount(etcd *druidv1alpha1.Etcd, disableAutomount bool) *corev1.ServiceAccount { - sa := emptyServiceAccount(getObjectKey(etcd)) + sa := emptyServiceAccount(getObjectKey(etcd.ObjectMeta)) buildResource(etcd, sa, !disableAutomount) return sa } func getLatestServiceAccount(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.ServiceAccount, error) { sa := &corev1.ServiceAccount{} - err := cl.Get(context.Background(), client.ObjectKey{Name: etcd.GetServiceAccountName(), Namespace: etcd.Namespace}, sa) + err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), Namespace: etcd.Namespace}, sa) return sa, err } func matchServiceAccount(g *WithT, etcd *druidv1alpha1.Etcd, actualSA corev1.ServiceAccount, disableAutoMount bool) { g.Expect(actualSA).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(etcd.GetServiceAccountName()), + "Name": Equal(druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta)), "Namespace": Equal(etcd.Namespace), - "Labels": testutils.MatchResourceLabels(etcd.GetDefaultLabels()), + "Labels": testutils.MatchResourceLabels(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)), "OwnerReferences": testutils.MatchEtcdOwnerReference(etcd.Name, etcd.UID), }), "AutomountServiceAccountToken": PointTo(Equal(!disableAutoMount)), diff --git a/internal/component/snapshotlease/snapshotlease.go b/internal/component/snapshotlease/snapshotlease.go index 4dd828c91..ab4207bf6 100644 --- a/internal/component/snapshotlease/snapshotlease.go +++ b/internal/component/snapshotlease/snapshotlease.go @@ -42,36 +42,36 @@ func New(client client.Client) component.Operator { } // GetExistingResourceNames returns the names of the existing snapshot leases for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 2) // We have to get snapshot leases one lease at a time and cannot use label-selector based listing // because currently snapshot lease do not have proper labels on them. In this new code // we will add the labels. // TODO: Once all snapshot leases have a purpose label on them, then we can use List instead of individual Get calls. - deltaSnapshotObjectKey := client.ObjectKey{Name: etcd.GetDeltaSnapshotLeaseName(), Namespace: etcd.Namespace} + deltaSnapshotObjectKey := client.ObjectKey{Name: druidv1alpha1.GetDeltaSnapshotLeaseName(etcdObjMeta), Namespace: etcdObjMeta.Namespace} deltaSnapshotLease, err := r.getLeasePartialObjectMetadata(ctx, deltaSnapshotObjectKey) if err != nil { return resourceNames, &druiderr.DruidError{ Code: ErrGetSnapshotLease, Cause: err, Operation: "GetExistingResourceNames", - Message: fmt.Sprintf("Error getting delta snapshot lease: %v for etcd: %v", deltaSnapshotObjectKey, etcd.GetNamespaceName()), + Message: fmt.Sprintf("Error getting delta snapshot lease: %v for etcd: %v", deltaSnapshotObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), } } - if deltaSnapshotLease != nil && metav1.IsControlledBy(deltaSnapshotLease, etcd) { + if deltaSnapshotLease != nil && metav1.IsControlledBy(deltaSnapshotLease, &etcdObjMeta) { resourceNames = append(resourceNames, deltaSnapshotLease.Name) } - fullSnapshotObjectKey := client.ObjectKey{Name: etcd.GetFullSnapshotLeaseName(), Namespace: etcd.Namespace} + fullSnapshotObjectKey := client.ObjectKey{Name: druidv1alpha1.GetFullSnapshotLeaseName(etcdObjMeta), Namespace: etcdObjMeta.Namespace} fullSnapshotLease, err := r.getLeasePartialObjectMetadata(ctx, fullSnapshotObjectKey) if err != nil { return resourceNames, &druiderr.DruidError{ Code: ErrGetSnapshotLease, Cause: err, Operation: "GetExistingResourceNames", - Message: fmt.Sprintf("Error getting full snapshot lease: %v for etcd: %v", fullSnapshotObjectKey, etcd.GetNamespaceName()), + Message: fmt.Sprintf("Error getting full snapshot lease: %v for etcd: %v", fullSnapshotObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), } } - if fullSnapshotLease != nil && metav1.IsControlledBy(fullSnapshotLease, etcd) { + if fullSnapshotLease != nil && metav1.IsControlledBy(fullSnapshotLease, &etcdObjMeta) { resourceNames = append(resourceNames, fullSnapshotLease.Name) } return resourceNames, nil @@ -82,11 +82,11 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { if !etcd.IsBackupStoreEnabled() { ctx.Logger.Info("Backup has been disabled. Triggering deletion of snapshot leases") - return r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { + return r.deleteAllSnapshotLeases(ctx, etcd.ObjectMeta, func(err error) error { return druiderr.WrapError(err, ErrSyncSnapshotLease, "Sync", - fmt.Sprintf("Failed to delete existing snapshot leases due to backup being disabled for etcd: %v", etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete existing snapshot leases due to backup being disabled for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) }) } @@ -106,13 +106,13 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) } // TriggerDelete triggers the deletion of the snapshot leases for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { ctx.Logger.Info("Triggering deletion of snapshot leases") - if err := r.deleteAllSnapshotLeases(ctx, etcd, func(err error) error { + if err := r.deleteAllSnapshotLeases(ctx, etcdObjMeta, func(err error) error { return druiderr.WrapError(err, ErrDeleteSnapshotLease, "TriggerDelete", - fmt.Sprintf("Failed to delete snapshot leases for etcd: %v", etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete snapshot leases for etcd: %v", druidv1alpha1.GetNamespaceName(etcdObjMeta))) }); err != nil { return err } @@ -120,11 +120,11 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return nil } -func (r _resource) deleteAllSnapshotLeases(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, wrapErrFn func(error) error) error { +func (r _resource) deleteAllSnapshotLeases(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta, wrapErrFn func(error) error) error { if err := r.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, - client.InNamespace(etcd.Namespace), - client.MatchingLabels(getSelectorLabelsForAllSnapshotLeases(etcd))); err != nil { + client.InNamespace(etcdObjMeta.Namespace), + client.MatchingLabels(getSelectorLabelsForAllSnapshotLeases(etcdObjMeta))); err != nil { return wrapErrFn(err) } return nil @@ -152,7 +152,7 @@ func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1 return druiderr.WrapError(err, ErrSyncSnapshotLease, "Sync", - fmt.Sprintf("Error syncing snapshot lease: %v for etcd: %v", leaseObjectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error syncing snapshot lease: %v for etcd: %v", leaseObjectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } ctx.Logger.Info("triggered create or update of snapshot lease", "objectKey", leaseObjectKey, "operationResult", opResult) @@ -161,14 +161,14 @@ func (r _resource) doCreateOrUpdate(ctx component.OperatorContext, etcd *druidv1 func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { lease.Labels = getLabels(etcd, lease.Name) - lease.OwnerReferences = []metav1.OwnerReference{etcd.GetAsOwnerReference()} + lease.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} } -func getSelectorLabelsForAllSnapshotLeases(etcd *druidv1alpha1.Etcd) map[string]string { +func getSelectorLabelsForAllSnapshotLeases(etcdObjMeta metav1.ObjectMeta) map[string]string { leaseMatchingLabels := map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, } - return utils.MergeMaps(etcd.GetDefaultLabels(), leaseMatchingLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcdObjMeta), leaseMatchingLabels) } func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { @@ -176,17 +176,17 @@ func getLabels(etcd *druidv1alpha1.Etcd, leaseName string) map[string]string { druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelAppNameKey: leaseName, } - return utils.MergeMaps(leaseLabels, etcd.GetDefaultLabels()) + return utils.MergeMaps(leaseLabels, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)) } func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { return []client.ObjectKey{ { - Name: etcd.GetDeltaSnapshotLeaseName(), + Name: druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta), Namespace: etcd.Namespace, }, { - Name: etcd.GetFullSnapshotLeaseName(), + Name: druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta), Namespace: etcd.Namespace, }, } diff --git a/internal/component/snapshotlease/snapshotlease_test.go b/internal/component/snapshotlease/snapshotlease_test.go index 58ab87458..e675d4358 100644 --- a/internal/component/snapshotlease/snapshotlease_test.go +++ b/internal/component/snapshotlease/snapshotlease_test.go @@ -80,7 +80,7 @@ func TestGetExistingResourceNames(t *testing.T) { cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKeys(etcd)...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - actualSnapshotLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd) + actualSnapshotLeaseNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -128,7 +128,7 @@ func TestSyncWhenBackupIsEnabled(t *testing.T) { g.Expect(latestSnapshotLeases).To(BeEmpty()) } else { g.Expect(listErr).ToNot(HaveOccurred()) - g.Expect(latestSnapshotLeases).To(ConsistOf(matchLease(etcd.GetDeltaSnapshotLeaseName(), etcd), matchLease(etcd.GetFullSnapshotLeaseName(), etcd))) + g.Expect(latestSnapshotLeases).To(ConsistOf(matchLease(druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta), etcd), matchLease(druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta), etcd))) } }) } @@ -167,7 +167,7 @@ func TestSyncWhenBackupHasBeenDisabled(t *testing.T) { newDeltaSnapshotLease(nonTargetEtcd), newFullSnapshotLease(nonTargetEtcd), } - cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, existingEtcd.Namespace, getSelectorLabelsForAllSnapshotLeases(existingEtcd), existingObjects...) + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllOfErr, nil, existingEtcd.Namespace, getSelectorLabelsForAllSnapshotLeases(existingEtcd.ObjectMeta), existingObjects...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) syncErr := operator.Sync(opCtx, updatedEtcd) @@ -229,10 +229,10 @@ func TestTriggerDelete(t *testing.T) { if tc.backupEnabled { existingObjects = append(existingObjects, newDeltaSnapshotLease(etcd), newFullSnapshotLease(etcd)) } - cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllErr, nil, etcd.Namespace, getSelectorLabelsForAllSnapshotLeases(etcd), existingObjects...) + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(tc.deleteAllErr, nil, etcd.Namespace, getSelectorLabelsForAllSnapshotLeases(etcd.ObjectMeta), existingObjects...) operator := New(cl) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - triggerDeleteErr := operator.TriggerDelete(opCtx, etcd) + triggerDeleteErr := operator.TriggerDelete(opCtx, etcd.ObjectMeta) latestSnapshotLeases, snapshotLeaseListErr := getLatestSnapshotLeases(cl, etcd) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, triggerDeleteErr) @@ -252,12 +252,12 @@ func TestTriggerDelete(t *testing.T) { // ---------------------------- Helper Functions ----------------------------- func newDeltaSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { - leaseName := etcd.GetDeltaSnapshotLeaseName() + leaseName := druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta) return buildLease(etcd, leaseName) } func newFullSnapshotLease(etcd *druidv1alpha1.Etcd) *coordinationv1.Lease { - leaseName := etcd.GetFullSnapshotLeaseName() + leaseName := druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta) return buildLease(etcd, leaseName) } @@ -266,17 +266,17 @@ func buildLease(etcd *druidv1alpha1.Etcd, leaseName string) *coordinationv1.Leas ObjectMeta: metav1.ObjectMeta{ Name: leaseName, Namespace: etcd.Namespace, - Labels: utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ + Labels: utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelAppNameKey: leaseName, }), - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + OwnerReferences: []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)}, }, } } func matchLease(leaseName string, etcd *druidv1alpha1.Etcd) gomegatypes.GomegaMatcher { - expectedLabels := utils.MergeMaps(etcd.GetDefaultLabels(), map[string]string{ + expectedLabels := utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelAppNameKey: leaseName, }) @@ -295,7 +295,7 @@ func getLatestSnapshotLeases(cl client.Client, etcd *druidv1alpha1.Etcd) ([]coor etcd, utils.MergeMaps(map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, - }, etcd.GetDefaultLabels())) + }, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta))) } func doGetLatestLeases(cl client.Client, etcd *druidv1alpha1.Etcd, matchingLabels map[string]string) ([]coordinationv1.Lease, error) { diff --git a/internal/component/statefulset/builder.go b/internal/component/statefulset/builder.go index 68a7ce3cf..4381ad1da 100644 --- a/internal/component/statefulset/builder.go +++ b/internal/component/statefulset/builder.go @@ -113,7 +113,7 @@ func (b *stsBuilder) createStatefulSetObjectMeta() { Name: b.etcd.Name, Namespace: b.etcd.Namespace, Labels: b.getStatefulSetLabels(), - OwnerReferences: []metav1.OwnerReference{b.etcd.GetAsOwnerReference()}, + OwnerReferences: []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(b.etcd.ObjectMeta)}, } } @@ -122,7 +122,7 @@ func (b *stsBuilder) getStatefulSetLabels() map[string]string { druidv1alpha1.LabelComponentKey: common.ComponentNameStatefulSet, druidv1alpha1.LabelAppNameKey: b.etcd.Name, } - return utils.MergeMaps(b.etcd.GetDefaultLabels(), stsLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(b.etcd.ObjectMeta), stsLabels) } func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error { @@ -139,12 +139,12 @@ func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error b.sts.Spec = appsv1.StatefulSetSpec{ Replicas: pointer.Int32(b.replicas), Selector: &metav1.LabelSelector{ - MatchLabels: b.etcd.GetDefaultLabels(), + MatchLabels: druidv1alpha1.GetDefaultLabels(b.etcd.ObjectMeta), }, PodManagementPolicy: defaultPodManagementPolicy, UpdateStrategy: defaultUpdateStrategy, VolumeClaimTemplates: b.getVolumeClaimTemplates(), - ServiceName: b.etcd.GetPeerServiceName(), + ServiceName: druidv1alpha1.GetPeerServiceName(b.etcd.ObjectMeta), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: utils.MergeMaps(b.etcd.Spec.Labels, b.getStatefulSetLabels()), @@ -152,7 +152,7 @@ func (b *stsBuilder) createStatefulSetSpec(ctx component.OperatorContext) error }, Spec: corev1.PodSpec{ HostAliases: b.getHostAliases(), - ServiceAccountName: b.etcd.GetServiceAccountName(), + ServiceAccountName: druidv1alpha1.GetServiceAccountName(b.etcd.ObjectMeta), ShareProcessNamespace: pointer.Bool(true), InitContainers: b.getPodInitContainers(), Containers: []corev1.Container{ @@ -433,12 +433,12 @@ func (b *stsBuilder) getBackupRestoreContainerCommandArgs() []string { commandArgs = append(commandArgs, "--insecure-transport=false") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=false") commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=https://%s-local:%d", b.etcd.Name, b.clientPort)) - commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=https://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=https://%s:%d", druidv1alpha1.GetClientServiceName(b.etcd.ObjectMeta), b.clientPort)) } else { commandArgs = append(commandArgs, "--insecure-transport=true") commandArgs = append(commandArgs, "--insecure-skip-tls-verify=true") commandArgs = append(commandArgs, fmt.Sprintf("--endpoints=http://%s-local:%d", b.etcd.Name, b.clientPort)) - commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", b.etcd.GetClientServiceName(), b.clientPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--service-endpoints=http://%s:%d", druidv1alpha1.GetClientServiceName(b.etcd.ObjectMeta), b.clientPort)) } if b.etcd.Spec.Backup.TLS != nil { commandArgs = append(commandArgs, fmt.Sprintf("--server-cert=%s/tls.crt", common.VolumeMountPathBackupRestoreServerTLS)) @@ -489,14 +489,14 @@ func (b *stsBuilder) getBackupStoreCommandArgs() []string { // Full snapshot command line args // ----------------------------------------------------------------------------------------------------------------- - commandArgs = append(commandArgs, fmt.Sprintf("--full-snapshot-lease-name=%s", b.etcd.GetFullSnapshotLeaseName())) + commandArgs = append(commandArgs, fmt.Sprintf("--full-snapshot-lease-name=%s", druidv1alpha1.GetFullSnapshotLeaseName(b.etcd.ObjectMeta))) if b.etcd.Spec.Backup.FullSnapshotSchedule != nil { commandArgs = append(commandArgs, fmt.Sprintf("--schedule=%s", *b.etcd.Spec.Backup.FullSnapshotSchedule)) } // Delta snapshot command line args // ----------------------------------------------------------------------------------------------------------------- - commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-lease-name=%s", b.etcd.GetDeltaSnapshotLeaseName())) + commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-lease-name=%s", druidv1alpha1.GetDeltaSnapshotLeaseName(b.etcd.ObjectMeta))) if b.etcd.Spec.Backup.DeltaSnapshotPeriod != nil { commandArgs = append(commandArgs, fmt.Sprintf("--delta-snapshot-period=%s", b.etcd.Spec.Backup.DeltaSnapshotPeriod.Duration.String())) } @@ -692,7 +692,7 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: b.etcd.GetConfigMapName(), + Name: druidv1alpha1.GetConfigMapName(b.etcd.ObjectMeta), }, Items: []corev1.KeyToPath{ { @@ -826,7 +826,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol case utils.Local: hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, b.client, b.logger, store, b.etcd.GetNamespace()) if err != nil { - return nil, fmt.Errorf("error getting host mount path for etcd: %v Err: %w", b.etcd.GetNamespaceName(), err) + return nil, fmt.Errorf("error getting host mount path for etcd: %v Err: %w", druidv1alpha1.GetNamespaceName(b.etcd.ObjectMeta), err) } hpt := corev1.HostPathDirectory @@ -841,7 +841,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol }, nil case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: if store.SecretRef == nil { - return nil, fmt.Errorf("etcd: %v, no secretRef configured for backup store", b.etcd.GetNamespaceName()) + return nil, fmt.Errorf("etcd: %v, no secretRef configured for backup store", druidv1alpha1.GetNamespaceName(b.etcd.ObjectMeta)) } return &corev1.Volume{ diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index c207cb3b0..30adf83bc 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -47,9 +47,9 @@ func New(client client.Client, imageVector imagevector.ImageVector, featureGates } // GetExistingResourceNames returns the name of the existing statefulset for the given Etcd. -func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) { +func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) { resourceNames := make([]string, 0, 1) - objectKey := getObjectKey(etcd) + objectKey := getObjectKey(etcdObjMeta) objMeta := &metav1.PartialObjectMetadata{} objMeta.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) if err := r.client.Get(ctx, objectKey, objMeta); err != nil { @@ -59,9 +59,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcd return nil, druiderr.WrapError(err, ErrGetStatefulSet, "GetExistingResourceNames", - fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } - if metav1.IsControlledBy(objMeta, etcd) { + if metav1.IsControlledBy(objMeta, &etcdObjMeta) { resourceNames = append(resourceNames, objMeta.Name) } return resourceNames, nil @@ -73,12 +73,12 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) existingSTS *appsv1.StatefulSet err error ) - objectKey := getObjectKey(etcd) - if existingSTS, err = r.getExistingStatefulSet(ctx, etcd); err != nil { + objectKey := getObjectKey(etcd.ObjectMeta) + if existingSTS, err = r.getExistingStatefulSet(ctx, etcd.ObjectMeta); err != nil { return druiderr.WrapError(err, ErrSyncStatefulSet, "Sync", - fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } // There is no StatefulSet present. Create one. if existingSTS == nil { @@ -91,16 +91,16 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncStatefulSet, "Sync", - fmt.Sprintf("Error while handling peer URL TLS change for StatefulSet: %v, etcd: %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Error while handling peer URL TLS change for StatefulSet: %v, etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } return r.createOrPatch(ctx, etcd) } // TriggerDelete triggers the deletion of the statefulset for the given Etcd. -func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { - objectKey := getObjectKey(etcd) +func (r _resource) TriggerDelete(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) error { + objectKey := getObjectKey(etcdObjMeta) ctx.Logger.Info("Triggering deletion of StatefulSet", "objectKey", objectKey) - if err := r.client.Delete(ctx, emptyStatefulSet(etcd)); err != nil { + if err := r.client.Delete(ctx, emptyStatefulSet(etcdObjMeta)); err != nil { if apierrors.IsNotFound(err) { ctx.Logger.Info("No StatefulSet found, Deletion is a No-Op", "objectKey", objectKey.Name) return nil @@ -108,7 +108,7 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp return druiderr.WrapError(err, ErrDeleteStatefulSet, "TriggerDelete", - fmt.Sprintf("Failed to delete StatefulSet: %v for etcd %v", objectKey, etcd.GetNamespaceName())) + fmt.Sprintf("Failed to delete StatefulSet: %v for etcd %v", objectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta))) } ctx.Logger.Info("deleted", "component", "statefulset", "objectKey", objectKey) return nil @@ -116,9 +116,9 @@ func (r _resource) TriggerDelete(ctx component.OperatorContext, etcd *druidv1alp // getExistingStatefulSet gets the existing statefulset if it exists. // If it is not found, it simply returns nil. Any other errors are returned as is. -func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { - sts := emptyStatefulSet(etcd) - if err := r.client.Get(ctx, getObjectKey(etcd), sts); err != nil { +func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, etcdObjMeta metav1.ObjectMeta) (*appsv1.StatefulSet, error) { + sts := emptyStatefulSet(etcdObjMeta) + if err := r.client.Get(ctx, getObjectKey(etcdObjMeta), sts); err != nil { if apierrors.IsNotFound(err) { return nil, nil } @@ -130,13 +130,13 @@ func (r _resource) getExistingStatefulSet(ctx component.OperatorContext, etcd *d // createOrPatchWithReplicas ensures that the StatefulSet is updated with all changes from passed in etcd but the replicas set on the StatefulSet // are taken from the passed in replicas and not from the etcd component. func (r _resource) createOrPatchWithReplicas(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, replicas int32) error { - desiredStatefulSet := emptyStatefulSet(etcd) + desiredStatefulSet := emptyStatefulSet(etcd.ObjectMeta) mutatingFn := func() error { if builder, err := newStsBuilder(r.client, ctx.Logger, etcd, replicas, r.useEtcdWrapper, r.imageVector, desiredStatefulSet); err != nil { return druiderr.WrapError(err, ErrSyncStatefulSet, "Sync", - fmt.Sprintf("Error initializing StatefulSet builder for etcd %v", etcd.GetNamespaceName())) + fmt.Sprintf("Error initializing StatefulSet builder for etcd %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } else { return builder.Build(ctx) } @@ -146,10 +146,10 @@ func (r _resource) createOrPatchWithReplicas(ctx component.OperatorContext, etcd return druiderr.WrapError(err, ErrSyncStatefulSet, "Sync", - fmt.Sprintf("Error creating or patching StatefulSet: %s for etcd: %v", desiredStatefulSet.Name, etcd.GetNamespaceName())) + fmt.Sprintf("Error creating or patching StatefulSet: %s for etcd: %v", desiredStatefulSet.Name, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } - ctx.Logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd), "operationResult", opResult) + ctx.Logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd.ObjectMeta), "operationResult", opResult) return nil } @@ -174,7 +174,7 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru } else { ctx.Logger.Info("Secret volume mounts to enable Peer URL TLS have already been mounted. Skipping patching StatefulSet with secret volume mounts.") } - return fmt.Errorf("peer URL TLS not enabled for #%d members for etcd: %v, requeuing reconcile request", *existingSts.Spec.Replicas, etcd.GetNamespaceName()) + return fmt.Errorf("peer URL TLS not enabled for #%d members for etcd: %v, requeuing reconcile request", *existingSts.Spec.Replicas, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)) } ctx.Logger.Info("Peer URL TLS has been enabled for all currently running members") return nil @@ -199,19 +199,19 @@ func isPeerTLSEnablementPending(peerTLSEnabledStatusFromMembers bool, etcd *drui return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } -func emptyStatefulSet(etcd *druidv1alpha1.Etcd) *appsv1.StatefulSet { +func emptyStatefulSet(obj metav1.ObjectMeta) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: etcd.Name, - Namespace: etcd.Namespace, - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + Name: obj.Name, + Namespace: obj.Namespace, + OwnerReferences: []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(obj)}, }, } } -func getObjectKey(etcd *druidv1alpha1.Etcd) client.ObjectKey { +func getObjectKey(obj metav1.ObjectMeta) client.ObjectKey { return client.ObjectKey{ - Name: etcd.Name, - Namespace: etcd.Namespace, + Name: obj.Name, + Namespace: obj.Namespace, } } diff --git a/internal/component/statefulset/statefulset_test.go b/internal/component/statefulset/statefulset_test.go index cdc91365b..08dcfbe2a 100644 --- a/internal/component/statefulset/statefulset_test.go +++ b/internal/component/statefulset/statefulset_test.go @@ -65,12 +65,12 @@ func TestGetExistingResourceNames(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var existingObjects []client.Object if tc.stsExists { - existingObjects = append(existingObjects, emptyStatefulSet(etcd)) + existingObjects = append(existingObjects, emptyStatefulSet(etcd.ObjectMeta)) } - cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, getObjectKey(etcd.ObjectMeta)) operator := New(cl, nil, nil) opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), uuid.NewString()) - actualStsNames, err := operator.GetExistingResourceNames(opCtx, etcd) + actualStsNames, err := operator.GetExistingResourceNames(opCtx, etcd.ObjectMeta) if tc.expectedErr != nil { testutils.CheckDruidError(g, tc.expectedErr, err) } else { @@ -116,7 +116,7 @@ func TestSyncWhenNoSTSExists(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // *************** Build test environment *************** etcd := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testutils.TestNamespace).WithReplicas(tc.replicas).Build() - cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, []client.Object{buildBackupSecret()}, getObjectKey(etcd)) + cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, []client.Object{buildBackupSecret()}, getObjectKey(etcd.ObjectMeta)) etcdImage, etcdBRImage, initContainerImage, err := utils.GetEtcdImages(etcd, iv, true) g.Expect(err).ToNot(HaveOccurred()) stsMatcher := NewStatefulSetMatcher(g, cl, etcd, tc.replicas, true, initContainerImage, etcdImage, etcdBRImage, pointer.String(utils.Local)) diff --git a/internal/component/statefulset/stsmatcher.go b/internal/component/statefulset/stsmatcher.go index a763a160c..f0779d6a8 100644 --- a/internal/component/statefulset/stsmatcher.go +++ b/internal/component/statefulset/stsmatcher.go @@ -94,13 +94,13 @@ func (s StatefulSetMatcher) matchSTSObjectMeta() gomegatypes.GomegaMatcher { func (s StatefulSetMatcher) matchSpec() gomegatypes.GomegaMatcher { return MatchFields(IgnoreExtras, Fields{ "Replicas": PointTo(Equal(s.replicas)), - "Selector": testutils.MatchSpecLabelSelector(s.etcd.GetDefaultLabels()), + "Selector": testutils.MatchSpecLabelSelector(druidv1alpha1.GetDefaultLabels(s.etcd.ObjectMeta)), "PodManagementPolicy": Equal(appsv1.ParallelPodManagement), "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), }), "VolumeClaimTemplates": s.matchVolumeClaimTemplates(), - "ServiceName": Equal(s.etcd.GetPeerServiceName()), + "ServiceName": Equal(druidv1alpha1.GetPeerServiceName(s.etcd.ObjectMeta)), "Template": s.matchPodTemplateSpec(), }) } @@ -143,7 +143,7 @@ func (s StatefulSetMatcher) matchPodSpec() gomegatypes.GomegaMatcher { // NOTE: currently this matcher does not check affinity and TSC since these are seldom used. If these are used in future then this matcher should be enhanced. return MatchFields(IgnoreExtras, Fields{ "HostAliases": s.matchPodHostAliases(), - "ServiceAccountName": Equal(s.etcd.GetServiceAccountName()), + "ServiceAccountName": Equal(druidv1alpha1.GetServiceAccountName(s.etcd.ObjectMeta)), "ShareProcessNamespace": PointTo(Equal(true)), "InitContainers": s.matchPodInitContainers(), "Containers": s.matchContainers(), @@ -415,7 +415,7 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { "VolumeSource": MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(s.etcd.GetConfigMapName()), + "Name": Equal(druidv1alpha1.GetConfigMapName(s.etcd.ObjectMeta)), }), "Items": HaveExactElements(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ "Key": Equal(etcdConfigFileName), diff --git a/internal/component/types.go b/internal/component/types.go index aad60fb20..2ed478ec8 100644 --- a/internal/component/types.go +++ b/internal/component/types.go @@ -9,6 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // OperatorContext holds the underline context.Context along with additional data that needs to be passed from one reconcile-step to another in a multistep reconciliation run. @@ -40,9 +41,9 @@ func (o *OperatorContext) SetLogger(logger logr.Logger) { // Operator manages one or more resources of a specific Kind which are provisioned for an etcd cluster. type Operator interface { // GetExistingResourceNames gets all resources that currently exist that this Operator manages. - GetExistingResourceNames(ctx OperatorContext, etcd *druidv1alpha1.Etcd) ([]string, error) + GetExistingResourceNames(ctx OperatorContext, etcdObjMeta metav1.ObjectMeta) ([]string, error) // TriggerDelete triggers the deletion of all resources that this Operator manages. - TriggerDelete(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error + TriggerDelete(ctx OperatorContext, etcdObjMeta metav1.ObjectMeta) error // Sync synchronizes all resources that this Operator manages. If a component does not exist then it will // create it. If there are changes in the owning Etcd resource that transpires changes to one or more resources // managed by this Operator then those component(s) will be either be updated or a deletion is triggered. diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 9d6fde80c..0ad891cb5 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -31,6 +31,7 @@ const ( defaultEventsThreshold = 1000000 defaultActiveDeadlineDuration = 3 * time.Hour defaultMetricsScrapeWaitDuration = 0 + defaultMinWorkers = 1 ) // Config contains configuration for the Compaction Controller. @@ -65,11 +66,7 @@ func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { // Validate validates the config. func (cfg *Config) Validate() error { - minWorkers := 0 - if cfg.EnableBackupCompaction { - minWorkers = 1 - } - if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, minWorkers, cfg.Workers); err != nil { + if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, defaultMinWorkers, cfg.Workers); err != nil { return err } if err := utils.MustBeGreaterThan(eventsThresholdFlagName, 0, cfg.EventsThreshold); err != nil { diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index 03c13a2c1..f35322038 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -102,12 +102,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { // Update metrics for currently running compaction job, if any job := &batchv1.Job{} - if err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job); err != nil { + compactionJobName := druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta) + if err := r.Get(ctx, types.NamespacedName{Name: compactionJobName, Namespace: etcd.Namespace}, job); err != nil { if !errors.IsNotFound(err) { // Error reading the object - requeue the request. return ctrl.Result{ RequeueAfter: 10 * time.Second, - }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", etcd.GetCompactionJobName(), etcd.Namespace, err) + }, fmt.Errorf("error while fetching compaction job with the name %v, in the namespace %v: %v", compactionJobName, etcd.Namespace, err) } logger.Info("No compaction job currently running", "namespace", etcd.Namespace) } @@ -134,7 +135,7 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd metricJobDurationSeconds.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededTrue, druidmetrics.EtcdNamespace: etcd.Namespace}).Observe(job.Status.CompletionTime.Time.Sub(job.Status.StartTime.Time).Seconds()) } if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { - logger.Error(err, "Couldn't delete the successful job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) + logger.Error(err, "Couldn't delete the successful job", "namespace", etcd.Namespace, "name", compactionJobName) return ctrl.Result{ RequeueAfter: 10 * time.Second, }, fmt.Errorf("error while deleting successful compaction job: %v", err) @@ -162,17 +163,18 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job fullLease := &coordinationv1.Lease{} - - if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetFullSnapshotLeaseName()), fullLease); err != nil { - logger.Error(err, "Couldn't fetch full snap lease", "namespace", etcd.Namespace, "name", etcd.GetFullSnapshotLeaseName()) + fullSnapshotLeaseName := druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta) + if err := r.Get(ctx, kutil.Key(etcd.Namespace, fullSnapshotLeaseName), fullLease); err != nil { + logger.Error(err, "Couldn't fetch full snap lease", "namespace", etcd.Namespace, "name", fullSnapshotLeaseName) return ctrl.Result{ RequeueAfter: 10 * time.Second, }, err } deltaLease := &coordinationv1.Lease{} - if err := r.Get(ctx, kutil.Key(etcd.Namespace, etcd.GetDeltaSnapshotLeaseName()), deltaLease); err != nil { - logger.Error(err, "Couldn't fetch delta snap lease", "namespace", etcd.Namespace, "name", etcd.GetDeltaSnapshotLeaseName()) + deltaSnapshotLeaseName := druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta) + if err := r.Get(ctx, kutil.Key(etcd.Namespace, deltaSnapshotLeaseName), deltaLease); err != nil { + logger.Error(err, "Couldn't fetch delta snap lease", "namespace", etcd.Namespace, "name", deltaSnapshotLeaseName) return ctrl.Result{ RequeueAfter: 10 * time.Second, }, err @@ -207,7 +209,7 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd // Reconcile job only when number of accumulated revisions over the last full snapshot is more than the configured threshold value via 'events-threshold' flag if diff >= r.config.EventsThreshold { - logger.Info("Creating etcd compaction job", "namespace", etcd.Namespace, "name", etcd.GetCompactionJobName()) + logger.Info("Creating etcd compaction job", "namespace", etcd.Namespace, "name", compactionJobName) job, err = r.createCompactionJob(ctx, logger, etcd) if err != nil { metricJobsTotal.With(prometheus.Labels{druidmetrics.LabelSucceeded: druidmetrics.ValueSucceededFalse, druidmetrics.EtcdNamespace: etcd.Namespace}).Inc() @@ -228,7 +230,7 @@ func (r *Reconciler) reconcileJob(ctx context.Context, logger logr.Logger, etcd func (r *Reconciler) delete(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { job := &batchv1.Job{} - err := r.Get(ctx, types.NamespacedName{Name: etcd.GetCompactionJobName(), Namespace: etcd.Namespace}, job) + err := r.Get(ctx, types.NamespacedName{Name: druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta), Namespace: etcd.Namespace}, job) if err != nil { if !errors.IsNotFound(err) { return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction job: %v", err) @@ -261,7 +263,7 @@ func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetCompactionJobName(), + Name: druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta), Namespace: etcd.Namespace, Labels: getLabels(etcd), OwnerReferences: []metav1.OwnerReference{ @@ -287,7 +289,7 @@ func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger }, Spec: v1.PodSpec{ ActiveDeadlineSeconds: pointer.Int64(int64(activeDeadlineSeconds)), - ServiceAccountName: etcd.GetServiceAccountName(), + ServiceAccountName: druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), RestartPolicy: v1.RestartPolicyNever, Containers: []v1.Container{{ Name: "compact-backup", @@ -343,13 +345,13 @@ func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { jobLabels := map[string]string{ - druidv1alpha1.LabelAppNameKey: etcd.GetCompactionJobName(), + druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta), druidv1alpha1.LabelComponentKey: common.ComponentNameCompactionJob, "networking.gardener.cloud/to-dns": "allowed", "networking.gardener.cloud/to-private-networks": "allowed", "networking.gardener.cloud/to-public-networks": "allowed", } - return utils.MergeMaps(etcd.GetDefaultLabels(), jobLabels) + return utils.MergeMaps(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), jobLabels) } func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featuregate.Feature]bool) ([]v1.VolumeMount, error) { @@ -449,8 +451,8 @@ func getCompactionJobArgs(etcd *druidv1alpha1.Etcd, metricsScrapeWaitDuration st command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") command = append(command, "--metrics-scrape-wait-duration="+metricsScrapeWaitDuration) command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--full-snapshot-lease-name="+etcd.GetFullSnapshotLeaseName()) - command = append(command, "--delta-snapshot-lease-name="+etcd.GetDeltaSnapshotLeaseName()) + command = append(command, "--full-snapshot-lease-name="+druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta)) + command = append(command, "--delta-snapshot-lease-name="+druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta)) var quota int64 = DefaultETCDQuota if etcd.Spec.Etcd.Quota != nil { diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 8df09325e..591629ac6 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -35,8 +35,8 @@ func (r *Reconciler) triggerDeletionFlow(ctx component.OperatorContext, logger l } func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } operators := r.operatorRegistry.AllOperators() @@ -47,7 +47,7 @@ func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjK deleteTasks = append(deleteTasks, utils.OperatorTask{ Name: fmt.Sprintf("triggerDeletionFlow-%s-component", kind), Fn: func(ctx component.OperatorContext) error { - return operator.TriggerDelete(ctx, etcd) + return operator.TriggerDelete(ctx, etcdPartialObjMeta.ObjectMeta) }, }) } @@ -59,14 +59,14 @@ func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjK } func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } operators := r.operatorRegistry.AllOperators() resourceNamesAwaitingCleanup := make([]string, 0, len(operators)) for _, operator := range operators { - existingResourceNames, err := operator.GetExistingResourceNames(ctx, etcd) + existingResourceNames, err := operator.GetExistingResourceNames(ctx, etcdPartialObjMeta.ObjectMeta) if err != nil { return ctrlutils.ReconcileWithError(err) } @@ -81,20 +81,24 @@ func (r *Reconciler) verifyNoResourcesAwaitCleanUp(ctx component.OperatorContext } func (r *Reconciler) removeFinalizer(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } + //etcd := &druidv1alpha1.Etcd{} + //if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + // return result + //} ctx.Logger.Info("Removing finalizer", "finalizerName", common.FinalizerName) - if err := controllerutils.RemoveFinalizers(ctx, r.client, etcd, common.FinalizerName); client.IgnoreNotFound(err) != nil { + if err := controllerutils.RemoveFinalizers(ctx, r.client, etcdPartialObjMeta, common.FinalizerName); client.IgnoreNotFound(err) != nil { return ctrlutils.ReconcileWithError(err) } return ctrlutils.ContinueReconcile() } func (r *Reconciler) recordDeletionStartOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeDelete); err != nil { @@ -105,8 +109,8 @@ func (r *Reconciler) recordDeletionStartOperation(ctx component.OperatorContext, } func (r *Reconciler) recordIncompleteDeletionOperation(ctx component.OperatorContext, logger logr.Logger, etcdObjKey client.ObjectKey, exitReconcileStepResult ctrlutils.ReconcileStepResult) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if err := r.lastOpErrRecorder.RecordErrors(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeDelete, exitReconcileStepResult.GetDescription(), exitReconcileStepResult.GetErrors()...); err != nil { diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 07292bcb1..f7b17a890 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -10,6 +10,7 @@ import ( ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -32,24 +33,33 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etc } func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if _, ok := etcd.Annotations[v1beta1constants.GardenerOperation]; ok { + if metav1.HasAnnotation(etcdPartialObjMeta.ObjectMeta, v1beta1constants.GardenerOperation) { ctx.Logger.Info("Removing operation annotation") - withOpAnnotation := etcd.DeepCopy() - delete(etcd.Annotations, v1beta1constants.GardenerOperation) - if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { + withOpAnnotation := etcdPartialObjMeta.DeepCopy() + delete(etcdPartialObjMeta.Annotations, v1beta1constants.GardenerOperation) + if err := r.client.Patch(ctx, etcdPartialObjMeta, client.MergeFrom(withOpAnnotation)); err != nil { ctx.Logger.Error(err, "failed to remove operation annotation") return ctrlutils.ReconcileWithError(err) } } + //if _, ok := etcd.Annotations[v1beta1constants.GardenerOperation]; ok { + // ctx.Logger.Info("Removing operation annotation") + // withOpAnnotation := etcd.DeepCopy() + // delete(etcd.Annotations, v1beta1constants.GardenerOperation) + // if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { + // ctx.Logger.Error(err, "failed to remove operation annotation") + // return ctrlutils.ReconcileWithError(err) + // } + //} return ctrlutils.ContinueReconcile() } func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } resourceOperators := r.getOrderedOperatorsForSync() @@ -65,7 +75,7 @@ func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey func (r *Reconciler) updateObservedGeneration(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } originalEtcd := etcd.DeepCopy() @@ -112,7 +122,7 @@ func (r *Reconciler) recordIncompleteReconcileOperation(ctx component.OperatorCo // - Reconciliation is not initiated if EnableEtcdSpecAutoReconcile is false and none of the relevant annotations are present. func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool { // Check if spec reconciliation has been suspended, if yes, then record the event and return false. - if suspendReconcileAnnotKey := etcd.GetSuspendEtcdSpecReconcileAnnotationKey(); suspendReconcileAnnotKey != nil { + if suspendReconcileAnnotKey := druidv1alpha1.GetSuspendEtcdSpecReconcileAnnotationKey(etcd.ObjectMeta); suspendReconcileAnnotKey != nil { r.recordEtcdSpecReconcileSuspension(etcd, *suspendReconcileAnnotKey) return false } diff --git a/internal/controller/etcd/reconcile_status.go b/internal/controller/etcd/reconcile_status.go index 27d245145..8a2457fe6 100644 --- a/internal/controller/etcd/reconcile_status.go +++ b/internal/controller/etcd/reconcile_status.go @@ -20,7 +20,7 @@ type mutateEtcdStatusFn func(ctx component.OperatorContext, etcd *druidv1alpha1. func (r *Reconciler) reconcileStatus(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } sLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileStatus").WithValues("runID", ctx.RunID) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 0d0f7e5da..991170d2e 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -23,11 +23,10 @@ import ( "github.com/gardener/etcd-druid/internal/images" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/go-logr/logr" - "github.com/google/uuid" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -90,11 +89,7 @@ type reconcileFn func(ctx component.OperatorContext, objectKey client.ObjectKey) // 3. Status Reconciliation: Always update the status of the Etcd component to reflect its current state. // 4. Scheduled Requeue: Requeue the reconciliation request after a defined period (EtcdStatusSyncPeriod) to maintain sync. func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - return result.ReconcileResult() - } - runID := uuid.New().String() + runID := string(controller.ReconcileIDFromContext(ctx)) operatorCtx := component.NewOperatorContext(ctx, r.logger, runID) if result := r.reconcileEtcdDeletion(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { return result.ReconcileResult() @@ -134,11 +129,11 @@ func createAndInitializeOperatorRegistry(client client.Client, config *Config, i } func (r *Reconciler) reconcileEtcdDeletion(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { - etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + etcdPartialObjMetadata := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjectKey, etcdPartialObjMetadata); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - if etcd.IsMarkedForDeletion() { + if druidv1alpha1.IsEtcdMarkedForDeletion(etcdPartialObjMetadata.ObjectMeta) { dLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "delete").WithValues("runId", ctx.RunID) ctx.SetLogger(dLog) return r.triggerDeletionFlow(ctx, dLog, etcdObjectKey) @@ -148,7 +143,7 @@ func (r *Reconciler) reconcileEtcdDeletion(ctx component.OperatorContext, etcdOb func (r *Reconciler) reconcileSpec(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} - if result := r.getLatestEtcd(ctx, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { return result } if r.canReconcileSpec(etcd) { @@ -158,13 +153,3 @@ func (r *Reconciler) reconcileSpec(ctx component.OperatorContext, etcdObjectKey } return ctrlutils.ContinueReconcile() } - -func (r *Reconciler) getLatestEtcd(ctx context.Context, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ctrlutils.ReconcileStepResult { - if err := r.client.Get(ctx, objectKey, etcd); err != nil { - if apierrors.IsNotFound(err) { - return ctrlutils.DoNotRequeue() - } - return ctrlutils.ReconcileWithError(err) - } - return ctrlutils.ContinueReconcile() -} diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index db1a278ad..372b10a23 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -12,10 +12,18 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) +// EmptyEtcdPartialObjectMetadata creates an empty PartialObjectMetadata for an Etcd object. +func EmptyEtcdPartialObjectMetadata() *metav1.PartialObjectMetadata { + etcdObjMetadata := &metav1.PartialObjectMetadata{} + etcdObjMetadata.SetGroupVersionKind(druidv1alpha1.SchemeGroupVersion.WithKind("Etcd")) + return etcdObjMetadata +} + // GetLatestEtcd returns the latest version of the Etcd object. func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.ObjectKey, etcd *druidv1alpha1.Etcd) ReconcileStepResult { if err := client.Get(ctx, objectKey, etcd); err != nil { @@ -27,6 +35,17 @@ func GetLatestEtcd(ctx context.Context, client client.Client, objectKey client.O return ContinueReconcile() } +// GetLatestEtcdPartialObjectMeta returns the latest version of the Etcd object metadata. +func GetLatestEtcdPartialObjectMeta(ctx context.Context, client client.Client, objectKey client.ObjectKey, etcdObjMetadata *metav1.PartialObjectMetadata) ReconcileStepResult { + if err := client.Get(ctx, objectKey, etcdObjMetadata); err != nil { + if apierrors.IsNotFound(err) { + return DoNotRequeue() + } + return ReconcileWithError(err) + } + return ContinueReconcile() +} + // ReconcileStepResult holds the result of a reconcile step. type ReconcileStepResult struct { result ctrl.Result diff --git a/internal/health/condition/check_data_volumes_ready_test.go b/internal/health/condition/check_data_volumes_ready_test.go index d02f4270d..d78f7d201 100644 --- a/internal/health/condition/check_data_volumes_ready_test.go +++ b/internal/health/condition/check_data_volumes_ready_test.go @@ -53,7 +53,7 @@ var _ = Describe("DataVolumesReadyCheck", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", - OwnerReferences: []metav1.OwnerReference{etcd.GetAsOwnerReference()}, + OwnerReferences: []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)}, }, Spec: appsv1.StatefulSetSpec{ Replicas: pointer.Int32(1), diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 22e54ac17..430c9aa5d 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -5,6 +5,8 @@ package controller import ( + "context" + "errors" "net" "net/http" "strconv" @@ -96,9 +98,22 @@ func createManager(config *Config) (ctrl.Manager, error) { func registerHealthAndReadyEndpoints(mgr ctrl.Manager, config *Config) error { slog.Info("Registering ping health check endpoint") + // Add a health check which always returns true when it is checked if err := mgr.AddHealthzCheck("ping", func(req *http.Request) error { return nil }); err != nil { return err } + // Add a readiness check which will pass only when all informers have synced. + if err := mgr.AddReadyzCheck("informer-sync", func(req *http.Request) error { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + if !mgr.GetCache().WaitForCacheSync(ctx) { + return errors.New("informers not synced yet") + } + return nil + }); err != nil { + return err + } + // Add a readiness check for the webhook server if config.Webhooks.AtLeaseOneEnabled() { slog.Info("Registering webhook-server readiness check endpoint") if err := mgr.AddReadyzCheck("webhook-server", mgr.GetWebhookServer().StartedChecker()); err != nil { diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index cbada7d0f..46045daf0 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -43,7 +43,7 @@ func IsStatefulSetReady(etcdReplicas int32, statefulSet *appsv1.StatefulSet) (bo // GetStatefulSet fetches StatefulSet created for the etcd. func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { statefulSets := &appsv1.StatefulSetList{} - if err := cl.List(ctx, statefulSets, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: labels.Set(etcd.GetDefaultLabels()).AsSelector()}); err != nil { + if err := cl.List(ctx, statefulSets, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: labels.Set(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)).AsSelector()}); err != nil { return nil, err } diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index 1448f839a..1c9c0aa8a 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" testutils "github.com/gardener/etcd-druid/test/utils" . "github.com/onsi/gomega" @@ -167,7 +168,7 @@ func TestGetStatefulSet(t *testing.T) { sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcdUID, etcd.Spec.Replicas) existingObjects = append(existingObjects, sts) } - cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, etcd.GetDefaultLabels(), existingObjects...) + cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), existingObjects...) foundSts, err := GetStatefulSet(context.Background(), cl, etcd) if tc.expectedErr != nil { g.Expect(err).To(HaveOccurred()) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 6f0e718a3..dfe5ee638 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -83,15 +83,28 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R // Leases (member and snapshot) will be periodically updated by etcd members. // Allow updates to such leases, but only by etcd members, which would use the serviceaccount deployed by druid for them. +<<<<<<< Updated upstream requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && req.Operation == admissionv1.Update { if serviceaccount.MatchesUsername(etcd.GetNamespace(), etcd.GetServiceAccountName(), req.UserInfo.Username) { +======= + if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && + req.Operation == admissionv1.Update { + saTokens := strings.Split(req.UserInfo.Username, ":") + saName := saTokens[len(saTokens)-1] // last element of sa name which looks like `system:serviceaccount::` + if saName == druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta) { +>>>>>>> Stashed changes return admission.Allowed("lease resource can be freely updated by etcd members") } } +<<<<<<< Updated upstream // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set. if !etcd.AreManagedResourcesProtected() { +======= + // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set + if !druidv1alpha1.AreManagedResourcesProtected(etcd.ObjectMeta) { +>>>>>>> Stashed changes return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index fb706d94a..1583edf7a 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -149,7 +149,7 @@ func TestHandleLeaseUpdate(t *testing.T) { username := testUserName if tc.useEtcdServiceAccount { - username = fmt.Sprintf("system:serviceaccount:%s:%s", testNamespace, etcd.GetServiceAccountName()) + username = fmt.Sprintf("system:serviceaccount:%s:%s", testNamespace, druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta)) } response := handler.Handle(context.Background(), admission.Request{ From befaada41ed9306b65af79919fcea390699a82ec Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 09:20:47 +0530 Subject: [PATCH 193/235] Makefile debug auto-reload is changed to manual Co-authored-by: Saketh Kalaga <51327242+renormalize@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ec2a8eec6..ae1f6bb4c 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ deploy: $(SKAFFOLD) $(HELM) .PHONY: deploy-dev deploy-dev: $(SKAFFOLD) $(HELM) $(SKAFFOLD) dev -m etcd-druid - + $(SKAFFOLD) dev -m etcd-druid --trigger='manual' .PHONY: deploy-debug deploy-debug: $(SKAFFOLD) $(HELM) $(SKAFFOLD) debug -m etcd-druid From 6ea3d898e17626686b605c5e5726430d7193bb9a Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 09:47:10 +0530 Subject: [PATCH 194/235] corrected makefile deploy-dev target and added doc string to explain readiness check for informers --- Makefile | 1 - internal/manager/manager.go | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ae1f6bb4c..fbe984072 100644 --- a/Makefile +++ b/Makefile @@ -161,7 +161,6 @@ deploy: $(SKAFFOLD) $(HELM) .PHONY: deploy-dev deploy-dev: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) dev -m etcd-druid $(SKAFFOLD) dev -m etcd-druid --trigger='manual' .PHONY: deploy-debug deploy-debug: $(SKAFFOLD) $(HELM) diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 430c9aa5d..826a797d2 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -102,7 +102,12 @@ func registerHealthAndReadyEndpoints(mgr ctrl.Manager, config *Config) error { if err := mgr.AddHealthzCheck("ping", func(req *http.Request) error { return nil }); err != nil { return err } + // Add a readiness check which will pass only when all informers have synced. + // Typically one would call `HasSync` but that is not exposed out of controller-runtime `cache.Informers`. Instead, + // give it a context with a very short timeout so that it causes the call to ` cache.WaitForCacheSync` to get executed once. + // We do not wish to wait longer as the readiness checks should be fast. Once all the cache informers have synced then the + // readiness check would succeed. if err := mgr.AddReadyzCheck("informer-sync", func(req *http.Request) error { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) defer cancel() @@ -113,6 +118,7 @@ func registerHealthAndReadyEndpoints(mgr ctrl.Manager, config *Config) error { }); err != nil { return err } + // Add a readiness check for the webhook server if config.Webhooks.AtLeaseOneEnabled() { slog.Info("Registering webhook-server readiness check endpoint") From 609bcdbaf309d8fd0cca248d713c17e38727605d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 09:50:26 +0530 Subject: [PATCH 195/235] resolved merge conflicts and adapted to use helper functions --- internal/webhook/sentinel/handler.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index dfe5ee638..4a104a2f0 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -83,28 +83,15 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R // Leases (member and snapshot) will be periodically updated by etcd members. // Allow updates to such leases, but only by etcd members, which would use the serviceaccount deployed by druid for them. -<<<<<<< Updated upstream requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && req.Operation == admissionv1.Update { - if serviceaccount.MatchesUsername(etcd.GetNamespace(), etcd.GetServiceAccountName(), req.UserInfo.Username) { -======= - if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && - req.Operation == admissionv1.Update { - saTokens := strings.Split(req.UserInfo.Username, ":") - saName := saTokens[len(saTokens)-1] // last element of sa name which looks like `system:serviceaccount::` - if saName == druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta) { ->>>>>>> Stashed changes + if serviceaccount.MatchesUsername(etcd.GetNamespace(), druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), req.UserInfo.Username) { return admission.Allowed("lease resource can be freely updated by etcd members") } } -<<<<<<< Updated upstream // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set. - if !etcd.AreManagedResourcesProtected() { -======= - // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set if !druidv1alpha1.AreManagedResourcesProtected(etcd.ObjectMeta) { ->>>>>>> Stashed changes return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } From d8d472bc7e8ffb8e36309e2961018d9873d4b69c Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 10:17:53 +0530 Subject: [PATCH 196/235] removed empty methods --- internal/manager/config.go | 8 +------- internal/webhook/config.go | 8 -------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/internal/manager/config.go b/internal/manager/config.go index 292bff3c1..d0efc9c70 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -159,7 +159,6 @@ func (cfg *Config) initFeatureGates(fs *flag.FlagSet) error { // captureFeatureActivations captures feature gate activations for etcd-druid controllers and webhooks. func (cfg *Config) captureFeatureActivations() { cfg.Controllers.CaptureFeatureActivations(cfg.FeatureGates) - cfg.Webhooks.CaptureFeatureActivations(cfg.FeatureGates) } // Validate validates the controller manager config. @@ -176,12 +175,7 @@ func (cfg *Config) Validate() error { return fmt.Errorf("webhook server cert dir cannot be empty") } } - - if err := cfg.Controllers.Validate(); err != nil { - return err - } - - return cfg.Webhooks.Validate() + return cfg.Controllers.Validate() } // getAllowedLeaderElectionResourceLocks returns the allowed resource type to be used for leader election. diff --git a/internal/webhook/config.go b/internal/webhook/config.go index 1b20ac561..7cac774b9 100644 --- a/internal/webhook/config.go +++ b/internal/webhook/config.go @@ -4,7 +4,6 @@ import ( "github.com/gardener/etcd-druid/internal/webhook/sentinel" flag "github.com/spf13/pflag" - "k8s.io/component-base/featuregate" ) // Config defines the configuration for etcd-druid webhooks. @@ -19,13 +18,6 @@ func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { sentinel.InitFromFlags(fs, cfg.Sentinel) } -// Validate validates the webhook config. -func (cfg *Config) Validate() error { return nil } - -// CaptureFeatureActivations captures feature gate activations for every webhook configuration. -// Feature gates are captured only for webhooks that use feature gates. -func (cfg *Config) CaptureFeatureActivations(_ featuregate.MutableFeatureGate) {} - // AtLeaseOneEnabled returns true if at least one webhook is enabled. // NOTE for contributors: For every new webhook, add a disjunction condition with the webhook's Enabled field. func (cfg *Config) AtLeaseOneEnabled() bool { From d7ba9db7358429aa6a4896dbac98878bae1fb807 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 11:58:50 +0530 Subject: [PATCH 197/235] fixed unit test --- api/v1alpha1/etcd_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api/v1alpha1/etcd_test.go b/api/v1alpha1/etcd_test.go index 823360abb..147eb1586 100644 --- a/api/v1alpha1/etcd_test.go +++ b/api/v1alpha1/etcd_test.go @@ -57,6 +57,7 @@ func TestIsReconciliationInProgress(t *testing.T) { { name: "when etcd status has lastOperation set and its state is Processing", lastOp: &LastOperation{ + Type: LastOperationTypeReconcile, State: LastOperationStateProcessing, }, expected: true, @@ -64,6 +65,7 @@ func TestIsReconciliationInProgress(t *testing.T) { { name: "when etcd status has lastOperation set and its state is Error", lastOp: &LastOperation{ + Type: LastOperationTypeReconcile, State: LastOperationStateError, }, expected: true, @@ -71,10 +73,27 @@ func TestIsReconciliationInProgress(t *testing.T) { { name: "when etcd status has lastOperation set and its state is Succeeded", lastOp: &LastOperation{ + Type: LastOperationTypeReconcile, State: LastOperationStateSucceeded, }, expected: false, }, + { + name: "when etcd status has lastOperation set and its type is delete", + lastOp: &LastOperation{ + Type: LastOperationTypeDelete, + State: LastOperationStateError, + }, + expected: false, + }, + { + name: "when etcd status has lastOperation set and its type is create", + lastOp: &LastOperation{ + Type: LastOperationTypeCreate, + State: LastOperationStateProcessing, + }, + expected: false, + }, { name: "when etcd status does not have lastOperation set", lastOp: nil, From 4b81cb48015e640bf2ef2f80328a7d403b1c6c7b Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 12:04:01 +0530 Subject: [PATCH 198/235] re-generated zz_generated.deepcopy.go --- api/v1alpha1/zz_generated.deepcopy.go | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8fd1cb4c3..a94f34e03 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -9,8 +9,8 @@ package v1alpha1 import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -39,12 +39,12 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } if in.Resources != nil { in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.CompactionResources != nil { in, out := &in.CompactionResources, &out.CompactionResources - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.FullSnapshotSchedule != nil { @@ -64,12 +64,12 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } if in.GarbageCollectionPeriod != nil { in, out := &in.GarbageCollectionPeriod, &out.GarbageCollectionPeriod - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.DeltaSnapshotPeriod != nil { in, out := &in.DeltaSnapshotPeriod, &out.DeltaSnapshotPeriod - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.DeltaSnapshotMemoryLimit != nil { @@ -79,7 +79,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } if in.DeltaSnapshotRetentionPeriod != nil { in, out := &in.DeltaSnapshotRetentionPeriod, &out.DeltaSnapshotRetentionPeriod - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.SnapshotCompression != nil { @@ -94,7 +94,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } if in.EtcdSnapshotTimeout != nil { in, out := &in.EtcdSnapshotTimeout, &out.EtcdSnapshotTimeout - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.LeaderElection != nil { @@ -257,7 +257,7 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { } if in.AuthSecretRef != nil { in, out := &in.AuthSecretRef, &out.AuthSecretRef - *out = new(v1.SecretReference) + *out = new(corev1.SecretReference) **out = **in } if in.Metrics != nil { @@ -267,7 +267,7 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { } if in.Resources != nil { in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.ClientUrlTLS != nil { @@ -282,12 +282,12 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { } if in.EtcdDefragTimeout != nil { in, out := &in.EtcdDefragTimeout, &out.EtcdDefragTimeout - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.HeartbeatDuration != nil { in, out := &in.HeartbeatDuration, &out.HeartbeatDuration - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.ClientService != nil { @@ -493,7 +493,7 @@ func (in *EtcdSpec) DeepCopyInto(out *EtcdSpec) { *out = *in if in.Selector != nil { in, out := &in.Selector, &out.Selector - *out = new(metav1.LabelSelector) + *out = new(v1.LabelSelector) (*in).DeepCopyInto(*out) } if in.Labels != nil { @@ -600,7 +600,7 @@ func (in *EtcdStatus) DeepCopyInto(out *EtcdStatus) { } if in.LabelSelector != nil { in, out := &in.LabelSelector, &out.LabelSelector - *out = new(metav1.LabelSelector) + *out = new(v1.LabelSelector) (*in).DeepCopyInto(*out) } if in.Members != nil { @@ -664,12 +664,12 @@ func (in *LeaderElectionSpec) DeepCopyInto(out *LeaderElectionSpec) { *out = *in if in.ReelectionPeriod != nil { in, out := &in.ReelectionPeriod, &out.ReelectionPeriod - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } if in.EtcdConnectionTimeout != nil { in, out := &in.EtcdConnectionTimeout, &out.EtcdConnectionTimeout - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } } @@ -689,12 +689,12 @@ func (in *SchedulingConstraints) DeepCopyInto(out *SchedulingConstraints) { *out = *in if in.Affinity != nil { in, out := &in.Affinity, &out.Affinity - *out = new(v1.Affinity) + *out = new(corev1.Affinity) (*in).DeepCopyInto(*out) } if in.TopologySpreadConstraints != nil { in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints - *out = make([]v1.TopologySpreadConstraint, len(*in)) + *out = make([]corev1.TopologySpreadConstraint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -772,7 +772,7 @@ func (in *StoreSpec) DeepCopyInto(out *StoreSpec) { } if in.SecretRef != nil { in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.SecretReference) + *out = new(corev1.SecretReference) **out = **in } } @@ -810,7 +810,7 @@ func (in *WaitForFinalSnapshotSpec) DeepCopyInto(out *WaitForFinalSnapshotSpec) *out = *in if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } } From 5ae9d94d257febab67944ca1cd41b5afd7c2cc5e Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 24 May 2024 12:15:41 +0530 Subject: [PATCH 199/235] fixed compilation issues in it tests, adapted recent changes to helper functions --- .../controllers/compaction/reconciler_test.go | 39 +++++++++---------- test/it/controller/etcd/assertions.go | 16 ++++---- test/it/controller/etcd/reconciler_test.go | 8 ++-- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index 2c6c960b9..e5311f4b5 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -13,7 +13,6 @@ import ( "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" - "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/test/matchers" . "github.com/onsi/ginkgo/v2" @@ -284,7 +283,7 @@ func validateEtcdForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.Job) Expect(*j).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(instance.GetCompactionJobName()), + "Name": Equal(druidv1alpha1.GetCompactionJobName(instance.ObjectMeta)), "Namespace": Equal(instance.Namespace), "OwnerReferences": MatchElements(testutils.OwnerRefIterator, IgnoreExtras, Elements{ instance.Name: MatchFields(IgnoreExtras, Fields{ @@ -305,19 +304,19 @@ func validateEtcdForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.Job) "Containers": MatchElements(testutils.ContainerIterator, IgnoreExtras, Elements{ "compact-backup": MatchFields(IgnoreExtras, Fields{ "Args": MatchElements(testutils.CmdIterator, IgnoreExtras, Elements{ - "--data-dir=/var/etcd/data/compaction.etcd": Equal("--data-dir=/var/etcd/data/compaction.etcd"), - "--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp"), - "--snapstore-temp-directory=/var/etcd/data/tmp": Equal("--snapstore-temp-directory=/var/etcd/data/tmp"), - "--metrics-scrape-wait-duration=1m0s": Equal("--metrics-scrape-wait-duration=1m0s"), - "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", instance.GetFullSnapshotLeaseName()): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", instance.GetFullSnapshotLeaseName())), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", instance.GetDeltaSnapshotLeaseName()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", instance.GetDeltaSnapshotLeaseName())), - fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), - fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), - fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container): Equal(fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container)), - fmt.Sprintf("--embedded-etcd-quota-bytes=%d", instance.Spec.Etcd.Quota.Value()): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", instance.Spec.Etcd.Quota.Value())), - fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", instance.Spec.Backup.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", instance.Spec.Backup.EtcdSnapshotTimeout.Duration.String())), - fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", instance.Spec.Etcd.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", instance.Spec.Etcd.EtcdDefragTimeout.Duration.String())), + "--data-dir=/var/etcd/data/compaction.etcd": Equal("--data-dir=/var/etcd/data/compaction.etcd"), + "--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/compaction.restoration.temp"), + "--snapstore-temp-directory=/var/etcd/data/tmp": Equal("--snapstore-temp-directory=/var/etcd/data/tmp"), + "--metrics-scrape-wait-duration=1m0s": Equal("--metrics-scrape-wait-duration=1m0s"), + "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), + fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", druidv1alpha1.GetFullSnapshotLeaseName(instance.ObjectMeta)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", druidv1alpha1.GetFullSnapshotLeaseName(instance.ObjectMeta))), + fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", druidv1alpha1.GetDeltaSnapshotLeaseName(instance.ObjectMeta)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", druidv1alpha1.GetDeltaSnapshotLeaseName(instance.ObjectMeta))), + fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), + fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), + fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container): Equal(fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container)), + fmt.Sprintf("--embedded-etcd-quota-bytes=%d", instance.Spec.Etcd.Quota.Value()): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", instance.Spec.Etcd.Quota.Value())), + fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", instance.Spec.Backup.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", instance.Spec.Backup.EtcdSnapshotTimeout.Duration.String())), + fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", instance.Spec.Etcd.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", instance.Spec.Etcd.EtcdDefragTimeout.Duration.String())), }), "Image": Equal(*instance.Spec.Backup.Image), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), @@ -680,7 +679,7 @@ func jobIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, job defer cancel() req := types.NamespacedName{ - Name: instance.GetCompactionJobName(), + Name: druidv1alpha1.GetCompactionJobName(instance.ObjectMeta), Namespace: instance.Namespace, } @@ -731,11 +730,11 @@ func deleteEtcdAndWait(c client.Client, etcd *druidv1alpha1.Etcd) { func createEtcdSnapshotLeasesAndWait(c client.Client, etcd *druidv1alpha1.Etcd) (*coordinationv1.Lease, *coordinationv1.Lease) { By("create full snapshot lease") - fullSnapLease := testutils.CreateLease(etcd.GetFullSnapshotLeaseName(), etcd.Namespace, etcd.Name, etcd.UID, "etcd-snapshot-lease") + fullSnapLease := testutils.CreateLease(druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta), etcd.Namespace, etcd.Name, etcd.UID, "etcd-snapshot-lease") Expect(c.Create(context.TODO(), fullSnapLease)).To(Succeed()) By("create delta snapshot lease") - deltaSnapLease := testutils.CreateLease(etcd.GetDeltaSnapshotLeaseName(), etcd.Namespace, etcd.Name, etcd.UID, "etcd-snapshot-lease") + deltaSnapLease := testutils.CreateLease(druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta), etcd.Namespace, etcd.Name, etcd.UID, "etcd-snapshot-lease") Expect(c.Create(context.TODO(), deltaSnapLease)).To(Succeed()) // wait for full snapshot lease to be created @@ -751,7 +750,7 @@ func deleteEtcdSnapshotLeasesAndWait(c client.Client, etcd *druidv1alpha1.Etcd) By("delete full snapshot lease") fullSnapLease := &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetFullSnapshotLeaseName(), + Name: druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta), Namespace: etcd.Namespace, }, } @@ -760,7 +759,7 @@ func deleteEtcdSnapshotLeasesAndWait(c client.Client, etcd *druidv1alpha1.Etcd) By("delete delta snapshot lease") deltaSnapLease := &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: etcd.GetDeltaSnapshotLeaseName(), + Name: druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta), Namespace: etcd.Namespace, }, } diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index c67b33492..0539d261d 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -118,7 +118,7 @@ func assertResourceCreation(ctx component.OperatorContext, t *testing.T, opRegis g := NewWithT(t) op := opRegistry.GetOperator(kind) checkFn := func() error { - actualResourceNames, err := op.GetExistingResourceNames(ctx, etcd) + actualResourceNames, err := op.GetExistingResourceNames(ctx, etcd.ObjectMeta) if err != nil { return err } @@ -150,23 +150,23 @@ func assertMemberLeasesCreated(ctx component.OperatorContext, t *testing.T, opRe func assertSnapshotLeasesCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { expectedSnapshotLeaseNames := make([]string, 0, 2) if etcd.IsBackupStoreEnabled() { - expectedSnapshotLeaseNames = []string{etcd.GetDeltaSnapshotLeaseName(), etcd.GetFullSnapshotLeaseName()} + expectedSnapshotLeaseNames = []string{druidv1alpha1.GetDeltaSnapshotLeaseName(etcd.ObjectMeta), druidv1alpha1.GetFullSnapshotLeaseName(etcd.ObjectMeta)} } assertResourceCreation(ctx, t, opRegistry, component.SnapshotLeaseKind, etcd, expectedSnapshotLeaseNames, timeout, pollInterval) } func assertClientServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { - expectedClientServiceNames := []string{etcd.GetClientServiceName()} + expectedClientServiceNames := []string{druidv1alpha1.GetClientServiceName(etcd.ObjectMeta)} assertResourceCreation(ctx, t, opRegistry, component.ClientServiceKind, etcd, expectedClientServiceNames, timeout, pollInterval) } func assertPeerServiceCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { - expectedPeerServiceNames := []string{etcd.GetPeerServiceName()} + expectedPeerServiceNames := []string{druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)} assertResourceCreation(ctx, t, opRegistry, component.PeerServiceKind, etcd, expectedPeerServiceNames, timeout, pollInterval) } func assertConfigMapCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { - expectedConfigMapNames := []string{etcd.GetConfigMapName()} + expectedConfigMapNames := []string{druidv1alpha1.GetConfigMapName(etcd.ObjectMeta)} assertResourceCreation(ctx, t, opRegistry, component.ConfigMapKind, etcd, expectedConfigMapNames, timeout, pollInterval) } @@ -176,17 +176,17 @@ func assertPDBCreated(ctx component.OperatorContext, t *testing.T, opRegistry co } func assertServiceAccountCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { - expectedServiceAccountNames := []string{etcd.GetServiceAccountName()} + expectedServiceAccountNames := []string{druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta)} assertResourceCreation(ctx, t, opRegistry, component.ServiceAccountKind, etcd, expectedServiceAccountNames, timeout, pollInterval) } func assertRoleCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { - expectedRoleNames := []string{etcd.GetRoleName()} + expectedRoleNames := []string{druidv1alpha1.GetRoleName(etcd.ObjectMeta)} assertResourceCreation(ctx, t, opRegistry, component.RoleKind, etcd, expectedRoleNames, timeout, pollInterval) } func assertRoleBindingCreated(ctx component.OperatorContext, t *testing.T, opRegistry component.Registry, etcd *druidv1alpha1.Etcd, timeout, pollInterval time.Duration) { - expectedRoleBindingNames := []string{etcd.GetRoleBindingName()} + expectedRoleBindingNames := []string{druidv1alpha1.GetRoleBindingName(etcd.ObjectMeta)} assertResourceCreation(ctx, t, opRegistry, component.RoleBindingKind, etcd, expectedRoleBindingNames, timeout, pollInterval) } diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 324069c49..a681a03bb 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -198,7 +198,7 @@ func testEtcdSpecUpdateWhenNoReconcileOperationAnnotationIsSet(t *testing.T, tes createAndAssertEtcdReconciliation(ctx, t, reconcilerTestEnv, etcdInstance) assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 1, 30*time.Second, 2*time.Second) // get updated version of etcdInstance - g.Expect(cl.Get(ctx, etcdInstance.GetNamespaceName(), etcdInstance)).To(Succeed()) + g.Expect(cl.Get(ctx, druidv1alpha1.GetNamespaceName(etcdInstance.ObjectMeta), etcdInstance)).To(Succeed()) // update etcdInstance spec without reconcile operation annotation set originalEtcdInstance := etcdInstance.DeepCopy() metricsLevelExtensive := druidv1alpha1.Extensive @@ -226,7 +226,7 @@ func testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet(t *testing.T, testN createAndAssertEtcdReconciliation(ctx, t, reconcilerTestEnv, etcdInstance) assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 1, 30*time.Second, 2*time.Second) // get latest version of etcdInstance - g.Expect(cl.Get(ctx, etcdInstance.GetNamespaceName(), etcdInstance)).To(Succeed()) + g.Expect(cl.Get(ctx, druidv1alpha1.GetNamespaceName(etcdInstance.ObjectMeta), etcdInstance)).To(Succeed()) // update etcdInstance spec with reconcile operation annotation also set originalEtcdInstance := etcdInstance.DeepCopy() metricsLevelExtensive := druidv1alpha1.Extensive @@ -308,7 +308,7 @@ func testPartialDeletionFailureOfEtcdResourcesWhenEtcdMarkedForDeletion(t *testi Build() // create the test client builder and record errors for delete operations for client service and snapshot lease. testClientBuilder := testutils.NewTestClientBuilder(). - RecordErrorForObjects(testutils.ClientMethodDelete, testutils.TestAPIInternalErr, client.ObjectKey{Name: etcdInstance.GetClientServiceName(), Namespace: etcdInstance.Namespace}). + RecordErrorForObjects(testutils.ClientMethodDelete, testutils.TestAPIInternalErr, client.ObjectKey{Name: druidv1alpha1.GetClientServiceName(etcdInstance.ObjectMeta), Namespace: etcdInstance.Namespace}). RecordErrorForObjectsMatchingLabels(testutils.ClientMethodDeleteAll, etcdInstance.Namespace, map[string]string{ druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotLease, druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, @@ -408,7 +408,7 @@ func TestEtcdStatusReconciliation(t *testing.T) { } func testConditionsAndMembersWhenAllMemberLeasesAreActive(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { - memberLeaseNames := etcd.GetMemberLeaseNames() + memberLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas)) testNs := etcd.Namespace clock := testclock.NewFakeClock(time.Now().Round(time.Second)) mlcs := []etcdMemberLeaseConfig{ From a13614e155c87c8f7b7cd8269a9d1f9a023c1c04 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 24 May 2024 14:11:53 +0530 Subject: [PATCH 200/235] Address review comments by @ashwani2k: move provider-specific code to dedicated package internal/store --- api/validation/etcd.go | 4 +- internal/component/statefulset/builder.go | 24 +++-- .../component/statefulset/statefulset_test.go | 4 +- internal/component/statefulset/stsmatcher.go | 16 ++-- internal/controller/compaction/reconciler.go | 32 ++++--- .../etcdcopybackupstask/reconciler.go | 31 ++++--- .../etcdcopybackupstask/reconciler_test.go | 92 +++++++++---------- internal/{utils => store}/store.go | 90 +++++++++++++----- internal/{utils => store}/store_test.go | 6 +- internal/utils/envvar.go | 51 +--------- test/e2e/etcd_backup_test.go | 12 ++- test/e2e/etcd_compaction_test.go | 13 +-- test/e2e/etcd_multi_node_test.go | 4 +- test/e2e/utils.go | 10 +- .../controllers/compaction/reconciler_test.go | 5 +- .../etcdcopybackupstask/reconciler_test.go | 6 +- 16 files changed, 213 insertions(+), 187 deletions(-) rename internal/{utils => store}/store.go (53%) rename internal/{utils => store}/store_test.go (96%) diff --git a/api/validation/etcd.go b/api/validation/etcd.go index 02e88ce27..b6fba4297 100644 --- a/api/validation/etcd.go +++ b/api/validation/etcd.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" apiequality "k8s.io/apimachinery/pkg/api/equality" apivalidation "k8s.io/apimachinery/pkg/api/validation" @@ -71,7 +71,7 @@ func validateStore(store *v1alpha1.StoreSpec, name, namespace string, path *fiel } if store.Provider != nil && *store.Provider != "" { - if _, err := utils.StorageProviderFromInfraProvider(store.Provider); err != nil { + if _, err := druidstore.StorageProviderFromInfraProvider(store.Provider); err != nil { allErrs = append(allErrs, field.Invalid(path.Child("provider"), store.Provider, err.Error())) } } diff --git a/internal/component/statefulset/builder.go b/internal/component/statefulset/builder.go index 4381ad1da..07f4a9448 100644 --- a/internal/component/statefulset/builder.go +++ b/internal/component/statefulset/builder.go @@ -12,8 +12,10 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/component" + druidstore "github.com/gardener/etcd-druid/internal/store" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -228,7 +230,7 @@ func (b *stsBuilder) getPodInitContainers() []corev1.Container { }, }) if b.etcd.IsBackupStoreEnabled() { - if b.provider != nil && *b.provider == utils.Local { + if b.provider != nil && *b.provider == druidstore.Local { etcdBackupVolumeMount := b.getEtcdBackupVolumeMount() if etcdBackupVolumeMount != nil { initContainers = append(initContainers, corev1.Container{ @@ -305,7 +307,7 @@ func (b *stsBuilder) getBackupRestoreContainerSecretVolumeMounts() []corev1.Volu func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { switch *b.provider { - case utils.Local: + case druidstore.Local: if b.etcd.Spec.Backup.Store.Container != nil { if b.useEtcdWrapper { return &corev1.VolumeMount{ @@ -319,12 +321,12 @@ func (b *stsBuilder) getEtcdBackupVolumeMount() *corev1.VolumeMount { } } } - case utils.GCS: + case druidstore.GCS: return &corev1.VolumeMount{ Name: common.VolumeNameProviderBackupSecret, MountPath: common.VolumeMountPathGCSBackupSecret, } - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + case druidstore.S3, druidstore.ABS, druidstore.OSS, druidstore.Swift, druidstore.OCS: return &corev1.VolumeMount{ Name: common.VolumeNameProviderBackupSecret, MountPath: common.VolumeMountPathNonGCSProviderBackupSecret, @@ -371,6 +373,12 @@ func (b *stsBuilder) getBackupRestoreContainer() (corev1.Container, error) { if err != nil { return corev1.Container{}, err } + providerEnv, err := druidstore.GetProviderEnvVars(b.etcd.Spec.Backup.Store) + if err != nil { + return corev1.Container{}, err + } + env = append(env, providerEnv...) + return corev1.Container{ Name: common.ContainerNameEtcdBackupRestore, Image: b.etcdBackupRestoreImage, @@ -823,8 +831,8 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol } store := b.etcd.Spec.Backup.Store switch *b.provider { - case utils.Local: - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, b.client, b.logger, store, b.etcd.GetNamespace()) + case druidstore.Local: + hostPath, err := druidstore.GetHostMountPathFromSecretRef(ctx, b.client, b.logger, store, b.etcd.GetNamespace()) if err != nil { return nil, fmt.Errorf("error getting host mount path for etcd: %v Err: %w", druidv1alpha1.GetNamespaceName(b.etcd.ObjectMeta), err) } @@ -839,7 +847,7 @@ func (b *stsBuilder) getBackupVolume(ctx component.OperatorContext) (*corev1.Vol }, }, }, nil - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + case druidstore.GCS, druidstore.S3, druidstore.OSS, druidstore.ABS, druidstore.Swift, druidstore.OCS: if store.SecretRef == nil { return nil, fmt.Errorf("etcd: %v, no secretRef configured for backup store", druidv1alpha1.GetNamespaceName(b.etcd.ObjectMeta)) } @@ -861,7 +869,7 @@ func getBackupStoreProvider(etcd *druidv1alpha1.Etcd) (*string, error) { if !etcd.IsBackupStoreEnabled() { return nil, nil } - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + provider, err := druidstore.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) if err != nil { return nil, err } diff --git a/internal/component/statefulset/statefulset_test.go b/internal/component/statefulset/statefulset_test.go index 08dcfbe2a..0d2e5b94a 100644 --- a/internal/component/statefulset/statefulset_test.go +++ b/internal/component/statefulset/statefulset_test.go @@ -13,8 +13,10 @@ import ( "github.com/gardener/etcd-druid/internal/component" druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" + druidstore "github.com/gardener/etcd-druid/internal/store" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" "github.com/google/uuid" . "github.com/onsi/gomega" @@ -119,7 +121,7 @@ func TestSyncWhenNoSTSExists(t *testing.T) { cl := testutils.CreateTestFakeClientForObjects(nil, tc.createErr, nil, nil, []client.Object{buildBackupSecret()}, getObjectKey(etcd.ObjectMeta)) etcdImage, etcdBRImage, initContainerImage, err := utils.GetEtcdImages(etcd, iv, true) g.Expect(err).ToNot(HaveOccurred()) - stsMatcher := NewStatefulSetMatcher(g, cl, etcd, tc.replicas, true, initContainerImage, etcdImage, etcdBRImage, pointer.String(utils.Local)) + stsMatcher := NewStatefulSetMatcher(g, cl, etcd, tc.replicas, true, initContainerImage, etcdImage, etcdBRImage, pointer.String(druidstore.Local)) operator := New(cl, iv, map[featuregate.Feature]bool{ features.UseEtcdWrapper: true, }) diff --git a/internal/component/statefulset/stsmatcher.go b/internal/component/statefulset/stsmatcher.go index f0779d6a8..fcca8b275 100644 --- a/internal/component/statefulset/stsmatcher.go +++ b/internal/component/statefulset/stsmatcher.go @@ -11,8 +11,10 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" + druidstore "github.com/gardener/etcd-druid/internal/store" "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -175,7 +177,7 @@ func (s StatefulSetMatcher) matchPodInitContainers() gomegatypes.GomegaMatcher { "SecurityContext": matchPodInitContainerSecurityContext(), }) initContainerMatcherElements[common.InitContainerNameChangePermissions] = changePermissionsInitContainerMatcher - if s.etcd.IsBackupStoreEnabled() && s.provider != nil && *s.provider == utils.Local { + if s.etcd.IsBackupStoreEnabled() && s.provider != nil && *s.provider == druidstore.Local { changeBackupBucketPermissionsMatcher := MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.InitContainerNameChangeBackupBucketPermissions), "Image": Equal(s.initContainerImage), @@ -363,7 +365,7 @@ func (s StatefulSetMatcher) getEtcdSecretVolMountsMatchers() []gomegatypes.Gomeg func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.GomegaMatcher { switch *s.provider { - case utils.Local: + case druidstore.Local: if s.etcd.Spec.Backup.Store.Container != nil { if s.useEtcdWrapper { return matchVolMount(common.VolumeNameLocalBackup, fmt.Sprintf("/home/nonroot/%s", pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, ""))) @@ -371,9 +373,9 @@ func (s StatefulSetMatcher) getEtcdBackupVolumeMountMatcher() gomegatypes.Gomega return matchVolMount(common.VolumeNameLocalBackup, pointer.StringDeref(s.etcd.Spec.Backup.Store.Container, "")) } } - case utils.GCS: + case druidstore.GCS: return matchVolMount(common.VolumeNameProviderBackupSecret, common.VolumeMountPathGCSBackupSecret) - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + case druidstore.S3, druidstore.ABS, druidstore.OSS, druidstore.Swift, druidstore.OCS: return matchVolMount(common.VolumeNameProviderBackupSecret, common.VolumeMountPathNonGCSProviderBackupSecret) } return nil @@ -530,8 +532,8 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { return nil } switch *s.provider { - case utils.Local: - hostPath, err := utils.GetHostMountPathFromSecretRef(context.Background(), s.cl, logr.Discard(), s.etcd.Spec.Backup.Store, s.etcd.Namespace) + case druidstore.Local: + hostPath, err := druidstore.GetHostMountPathFromSecretRef(context.Background(), s.cl, logr.Discard(), s.etcd.Spec.Backup.Store, s.etcd.Namespace) s.g.Expect(err).ToNot(HaveOccurred()) return MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.VolumeNameLocalBackup), @@ -543,7 +545,7 @@ func (s StatefulSetMatcher) getBackupVolumeMatcher() gomegatypes.GomegaMatcher { }), }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + case druidstore.GCS, druidstore.S3, druidstore.OSS, druidstore.ABS, druidstore.Swift, druidstore.OCS: s.g.Expect(s.etcd.Spec.Backup.Store.SecretRef).ToNot(BeNil()) return MatchFields(IgnoreExtras, Fields{ "Name": Equal(common.VolumeNameProviderBackupSecret), diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index f35322038..f6f6ea86c 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -15,6 +15,7 @@ import ( "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/images" druidmetrics "github.com/gardener/etcd-druid/internal/metrics" + druidstore "github.com/gardener/etcd-druid/internal/store" "github.com/gardener/etcd-druid/internal/utils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -311,14 +312,21 @@ func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger job.Spec.Template.Spec.Containers[0].VolumeMounts = vms } - if env, err := utils.GetBackupRestoreContainerEnvVars(etcd.Spec.Backup.Store); err != nil { - return nil, fmt.Errorf("error while creating compaction job in %v for %v : %v", + env, err := utils.GetBackupRestoreContainerEnvVars(etcd.Spec.Backup.Store) + if err != nil { + return nil, fmt.Errorf("error while creating compaction job in %v for %v : unable to get backup-restore container environment variables : %v", + etcd.Namespace, + etcd.Name, + err) + } + providerEnv, err := druidstore.GetProviderEnvVars(etcd.Spec.Backup.Store) + if err != nil { + return nil, fmt.Errorf("error while creating compaction job in %v for %v : unable to get provider-specific environment variables : %v", etcd.Namespace, etcd.Name, err) - } else { - job.Spec.Template.Spec.Containers[0].Env = env } + job.Spec.Template.Spec.Containers[0].Env = append(env, providerEnv...) if vm, err := getCompactionJobVolumes(ctx, r.Client, r.logger, etcd); err != nil { return nil, fmt.Errorf("error creating compaction job in %v for %v : %v", @@ -362,12 +370,12 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu }, } - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + provider, err := druidstore.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) if err != nil { return vms, fmt.Errorf("storage provider is not recognized while fetching volume mounts") } switch provider { - case utils.Local: + case druidstore.Local: if featureMap[features.UseEtcdWrapper] { vms = append(vms, v1.VolumeMount{ Name: "host-storage", @@ -379,12 +387,12 @@ func getCompactionJobVolumeMounts(etcd *druidv1alpha1.Etcd, featureMap map[featu MountPath: pointer.StringDeref(etcd.Spec.Backup.Store.Container, ""), }) } - case utils.GCS: + case druidstore.GCS: vms = append(vms, v1.VolumeMount{ Name: common.VolumeNameProviderBackupSecret, MountPath: common.VolumeMountPathGCSBackupSecret, }) - case utils.S3, utils.ABS, utils.OSS, utils.Swift, utils.OCS: + case druidstore.S3, druidstore.ABS, druidstore.OSS, druidstore.Swift, druidstore.OCS: vms = append(vms, v1.VolumeMount{ Name: common.VolumeNameProviderBackupSecret, MountPath: common.VolumeMountPathNonGCSProviderBackupSecret, @@ -405,13 +413,13 @@ func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr. } storeValues := etcd.Spec.Backup.Store - provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + provider, err := druidstore.StorageProviderFromInfraProvider(storeValues.Provider) if err != nil { return vs, fmt.Errorf("could not recognize storage provider while fetching volumes") } switch provider { case "Local": - hostPath, err := utils.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, etcd.Namespace) + hostPath, err := druidstore.GetHostMountPathFromSecretRef(ctx, cl, logger, storeValues, etcd.Namespace) if err != nil { return vs, fmt.Errorf("could not determine host mount path for local provider") } @@ -426,7 +434,7 @@ func getCompactionJobVolumes(ctx context.Context, cl client.Client, logger logr. }, }, }) - case utils.GCS, utils.S3, utils.OSS, utils.ABS, utils.Swift, utils.OCS: + case druidstore.GCS, druidstore.S3, druidstore.OSS, druidstore.ABS, druidstore.Swift, druidstore.OCS: if storeValues.SecretRef == nil { return vs, fmt.Errorf("could not configure secretRef for backup store %v", provider) } @@ -470,7 +478,7 @@ func getCompactionJobArgs(etcd *druidv1alpha1.Etcd, metricsScrapeWaitDuration st } storeValues := etcd.Spec.Backup.Store if storeValues != nil { - provider, err := utils.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) + provider, err := druidstore.StorageProviderFromInfraProvider(etcd.Spec.Backup.Store.Provider) if err == nil { command = append(command, "--storage-provider="+provider) } diff --git a/internal/controller/etcdcopybackupstask/reconciler.go b/internal/controller/etcdcopybackupstask/reconciler.go index 91f3a2ebd..d2aa007f1 100644 --- a/internal/controller/etcdcopybackupstask/reconciler.go +++ b/internal/controller/etcdcopybackupstask/reconciler.go @@ -14,6 +14,7 @@ import ( "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/images" + druidstore "github.com/gardener/etcd-druid/internal/store" "github.com/gardener/etcd-druid/internal/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -287,13 +288,13 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et } targetStore := task.Spec.TargetStore - targetProvider, err := utils.StorageProviderFromInfraProvider(targetStore.Provider) + targetProvider, err := druidstore.StorageProviderFromInfraProvider(targetStore.Provider) if err != nil { return nil, err } sourceStore := task.Spec.SourceStore - sourceProvider, err := utils.StorageProviderFromInfraProvider(sourceStore.Provider) + sourceProvider, err := druidstore.StorageProviderFromInfraProvider(sourceStore.Provider) if err != nil { return nil, err } @@ -353,7 +354,7 @@ func (r *Reconciler) createJobObject(ctx context.Context, task *druidv1alpha1.Et } if r.Config.FeatureGates[features.UseEtcdWrapper] { - if targetProvider == utils.Local { + if targetProvider == druidstore.Local { // init container to change file permissions of the folders used as store to 65532 (nonroot) // used only with local provider job.Spec.Template.Spec.InitContainers = []corev1.Container{ @@ -439,9 +440,9 @@ func getVolumeNamePrefix(prefix string) string { // This function creates the necessary Volume configurations for various storage providers. func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1alpha1.StoreSpec, namespace, provider, prefix string) (volumes []corev1.Volume, err error) { switch provider { - case utils.Local: + case druidstore.Local: hostPathDirectory := corev1.HostPathDirectory - hostPathPrefix, err := utils.GetHostMountPathFromSecretRef(ctx, r.Client, r.logger, store, namespace) + hostPathPrefix, err := druidstore.GetHostMountPathFromSecretRef(ctx, r.Client, r.logger, store, namespace) if err != nil { return nil, err } @@ -454,7 +455,7 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a }, }, }) - case utils.GCS, utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: + case druidstore.GCS, druidstore.S3, druidstore.ABS, druidstore.Swift, druidstore.OCS, druidstore.OSS: if store.SecretRef == nil { err = fmt.Errorf("no secretRef is configured for backup %sstore", prefix) return @@ -478,7 +479,7 @@ func (r *Reconciler) createVolumesFromStore(ctx context.Context, store *druidv1a // This function creates the necessary Volume configurations for various storage providers and returns any errors encountered. func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volumeMountPrefix string, useEtcdWrapper bool) (volumeMounts []corev1.VolumeMount) { switch provider { - case utils.Local: + case druidstore.Local: if useEtcdWrapper { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: volumeMountPrefix + "host-storage", @@ -490,12 +491,12 @@ func createVolumeMountsFromStore(store *druidv1alpha1.StoreSpec, provider, volum MountPath: *store.Container, }) } - case utils.GCS: + case druidstore.GCS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + common.VolumeNameProviderBackupSecret, MountPath: getGCSSecretVolumeMountPathWithPrefixAndSuffix(getVolumeNamePrefix(volumeMountPrefix), "/"), }) - case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: + case druidstore.S3, druidstore.ABS, druidstore.Swift, druidstore.OCS, druidstore.OSS: volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: getVolumeNamePrefix(volumeMountPrefix) + common.VolumeNameProviderBackupSecret, MountPath: getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/"), @@ -523,17 +524,17 @@ func getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, volumeSuffix s func createEnvVarsFromStore(store *druidv1alpha1.StoreSpec, storeProvider, envKeyPrefix, volumePrefix string) (envVars []corev1.EnvVar) { envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvStorageContainer, *store.Container)) switch storeProvider { - case utils.S3: + case druidstore.S3: envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAWSApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) - case utils.ABS: + case druidstore.ABS: envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAzureApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) - case utils.GCS: + case druidstore.GCS: envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvGoogleApplicationCredentials, getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"))) - case utils.Swift: + case druidstore.Swift: envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenstackApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) - case utils.OCS: + case druidstore.OCS: envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvOpenshiftApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) - case utils.OSS: + case druidstore.OSS: envVars = append(envVars, utils.GetEnvVarFromValue(envKeyPrefix+common.EnvAlicloudApplicationCredentials, getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""))) } return envVars diff --git a/internal/controller/etcdcopybackupstask/reconciler_test.go b/internal/controller/etcdcopybackupstask/reconciler_test.go index fedf64b19..5a82d2d5c 100644 --- a/internal/controller/etcdcopybackupstask/reconciler_test.go +++ b/internal/controller/etcdcopybackupstask/reconciler_test.go @@ -12,7 +12,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/client/kubernetes" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -314,8 +314,8 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Describe("#createJobArguments", func() { var ( - providerLocal = druidv1alpha1.StorageProvider(utils.Local) - providerS3 = druidv1alpha1.StorageProvider(utils.S3) + providerLocal = druidv1alpha1.StorageProvider(druidstore.Local) + providerS3 = druidv1alpha1.StorageProvider(druidstore.S3) task *druidv1alpha1.EtcdCopyBackupsTask expected = []string{ "copy", @@ -350,19 +350,19 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { }) It("should create the correct arguments", func() { - arguments := createJobArgs(task, utils.Local, utils.S3) + arguments := createJobArgs(task, druidstore.Local, druidstore.S3) Expect(arguments).To(Equal(expected)) }) It("should include the max backup age in the arguments", func() { task.Spec.MaxBackupAge = pointer.Uint32(10) - arguments := createJobArgs(task, utils.Local, utils.S3) + arguments := createJobArgs(task, druidstore.Local, druidstore.S3) Expect(arguments).To(Equal(append(expected, "--max-backup-age=10"))) }) It("should include the max number of backups in the arguments", func() { task.Spec.MaxBackups = pointer.Uint32(5) - arguments := createJobArgs(task, utils.Local, utils.S3) + arguments := createJobArgs(task, druidstore.Local, druidstore.S3) Expect(arguments).To(Equal(append(expected, "--max-backups-to-copy=5"))) }) @@ -370,7 +370,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { task.Spec.WaitForFinalSnapshot = &druidv1alpha1.WaitForFinalSnapshotSpec{ Enabled: true, } - arguments := createJobArgs(task, utils.Local, utils.S3) + arguments := createJobArgs(task, druidstore.Local, druidstore.S3) Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true"))) }) @@ -379,7 +379,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { Enabled: true, Timeout: &metav1.Duration{Duration: time.Minute}, } - arguments := createJobArgs(task, utils.Local, utils.S3) + arguments := createJobArgs(task, druidstore.Local, druidstore.S3) Expect(arguments).To(Equal(append(expected, "--wait-for-final-snapshot=true", "--wait-for-final-snapshot-timeout=1m0s"))) }) }) @@ -393,12 +393,12 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { ) // Loop through different storage providers to test with for _, p := range []string{ - utils.ABS, - utils.GCS, - utils.S3, - utils.Swift, - utils.OSS, - utils.OCS, + druidstore.ABS, + druidstore.GCS, + druidstore.S3, + druidstore.Swift, + druidstore.OSS, + druidstore.OCS, } { Context(fmt.Sprintf("with provider #%s", p), func() { provider := p @@ -419,7 +419,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { } Context("with provider #Local", func() { BeforeEach(func() { - storageProvider := druidv1alpha1.StorageProvider(utils.Local) + storageProvider := druidv1alpha1.StorageProvider(druidstore.Local) storeSpec = &druidv1alpha1.StoreSpec{ Container: &container, Provider: &storageProvider, @@ -427,8 +427,8 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { }) It("should create the correct env vars", func() { - envVars := createEnvVarsFromStore(storeSpec, utils.Local, envKeyPrefix, volumePrefix) - checkEnvVars(envVars, utils.Local, container, envKeyPrefix, volumePrefix) + envVars := createEnvVarsFromStore(storeSpec, druidstore.Local, envKeyPrefix, volumePrefix) + checkEnvVars(envVars, druidstore.Local, container, envKeyPrefix, volumePrefix) }) }) @@ -441,13 +441,13 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { ) // Loop through different storage providers to test with for _, p := range []string{ - utils.Local, - utils.ABS, - utils.GCS, - utils.S3, - utils.Swift, - utils.OSS, - utils.OCS, + druidstore.Local, + druidstore.ABS, + druidstore.GCS, + druidstore.S3, + druidstore.Swift, + druidstore.OSS, + druidstore.OCS, } { Context(fmt.Sprintf("with provider #%s", p), func() { provider := p @@ -467,13 +467,13 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { expectedMountName := "" switch provider { - case utils.Local: + case druidstore.Local: expectedMountName = volumeMountPrefix + "host-storage" expectedMountPath = *storeSpec.Container - case utils.GCS: + case druidstore.GCS: expectedMountName = volumeMountPrefix + common.VolumeNameProviderBackupSecret expectedMountPath = getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") - case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: + case druidstore.S3, druidstore.ABS, druidstore.Swift, druidstore.OCS, druidstore.OSS: expectedMountName = volumeMountPrefix + common.VolumeNameProviderBackupSecret expectedMountPath = getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumeMountPrefix, "/") default: @@ -522,7 +522,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { It("should create the correct volumes when secret data hostPath is set", func() { secret.Data = map[string][]byte{ - utils.EtcdBackupSecretHostPath: []byte("/test/hostPath"), + druidstore.EtcdBackupSecretHostPath: []byte("/test/hostPath"), } Expect(fakeClient.Create(ctx, secret)).To(Succeed()) @@ -549,7 +549,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { hostPathVolumeSource := volumes[0].VolumeSource.HostPath Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal(utils.LocalProviderDefaultMountPath + "/" + *store.Container)) + Expect(hostPathVolumeSource.Path).To(Equal(druidstore.LocalProviderDefaultMountPath + "/" + *store.Container)) Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) }) @@ -564,7 +564,7 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { hostPathVolumeSource := volumes[0].VolumeSource.HostPath Expect(hostPathVolumeSource).NotTo(BeNil()) - Expect(hostPathVolumeSource.Path).To(Equal(utils.LocalProviderDefaultMountPath + "/" + *store.Container)) + Expect(hostPathVolumeSource.Path).To(Equal(druidstore.LocalProviderDefaultMountPath + "/" + *store.Container)) Expect(*hostPathVolumeSource.Type).To(Equal(corev1.HostPathDirectory)) }) }) @@ -585,12 +585,12 @@ var _ = Describe("EtcdCopyBackupsTaskController", func() { // Loop through different storage providers to test with for _, p := range []string{ - utils.ABS, - utils.GCS, - utils.S3, - utils.Swift, - utils.OSS, - utils.OCS, + druidstore.ABS, + druidstore.GCS, + druidstore.S3, + druidstore.Swift, + druidstore.OSS, + druidstore.OCS, } { Context(fmt.Sprintf("#%s", p), func() { BeforeEach(func() { @@ -696,20 +696,20 @@ func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefi Value: container, }} mapToEnvVarKey := map[string]string{ - utils.S3: envKeyPrefix + common.EnvAWSApplicationCredentials, - utils.ABS: envKeyPrefix + common.EnvAzureApplicationCredentials, - utils.GCS: envKeyPrefix + common.EnvGoogleApplicationCredentials, - utils.Swift: envKeyPrefix + common.EnvOpenstackApplicationCredentials, - utils.OCS: envKeyPrefix + common.EnvOpenshiftApplicationCredentials, - utils.OSS: envKeyPrefix + common.EnvAlicloudApplicationCredentials, + druidstore.S3: envKeyPrefix + common.EnvAWSApplicationCredentials, + druidstore.ABS: envKeyPrefix + common.EnvAzureApplicationCredentials, + druidstore.GCS: envKeyPrefix + common.EnvGoogleApplicationCredentials, + druidstore.Swift: envKeyPrefix + common.EnvOpenstackApplicationCredentials, + druidstore.OCS: envKeyPrefix + common.EnvOpenshiftApplicationCredentials, + druidstore.OSS: envKeyPrefix + common.EnvAlicloudApplicationCredentials, } switch storeProvider { - case utils.S3, utils.ABS, utils.Swift, utils.OCS, utils.OSS: + case druidstore.S3, druidstore.ABS, druidstore.Swift, druidstore.OCS, druidstore.OSS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], Value: getNonGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, ""), }) - case utils.GCS: + case druidstore.GCS: expected = append(expected, corev1.EnvVar{ Name: mapToEnvVarKey[storeProvider], Value: getGCSSecretVolumeMountPathWithPrefixAndSuffix(volumePrefix, "/serviceaccount.json"), @@ -719,9 +719,9 @@ func checkEnvVars(envVars []corev1.EnvVar, storeProvider, container, envKeyPrefi } func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.ImageVector) gomegatypes.GomegaMatcher { - sourceProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.SourceStore.Provider) + sourceProvider, err := druidstore.StorageProviderFromInfraProvider(task.Spec.SourceStore.Provider) Expect(err).NotTo(HaveOccurred()) - targetProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) + targetProvider, err := druidstore.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) Expect(err).NotTo(HaveOccurred()) images, err := imagevector.FindImages(imageVector, []string{common.ImageKeyEtcdBackupRestore}) diff --git a/internal/utils/store.go b/internal/store/store.go similarity index 53% rename from internal/utils/store.go rename to internal/store/store.go index 924bc45f6..7eb282dc1 100644 --- a/internal/utils/store.go +++ b/internal/store/store.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package utils +package store import ( "context" @@ -10,6 +10,9 @@ import ( "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/utils" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -51,26 +54,6 @@ const ( OCS = "OCS" ) -// GetHostMountPathFromSecretRef returns the hostPath configured for the given store. -func GetHostMountPathFromSecretRef(ctx context.Context, client client.Client, logger logr.Logger, store *druidv1alpha1.StoreSpec, namespace string) (string, error) { - if store.SecretRef == nil { - logger.Info("secretRef is not defined for store, using default hostPath", "namespace", namespace) - return LocalProviderDefaultMountPath, nil - } - - secret := &corev1.Secret{} - if err := client.Get(ctx, Key(namespace, store.SecretRef.Name), secret); err != nil { - return "", err - } - - hostPath, ok := secret.Data[EtcdBackupSecretHostPath] - if !ok { - return LocalProviderDefaultMountPath, nil - } - - return string(hostPath), nil -} - // StorageProviderFromInfraProvider converts infra to object store provider. func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (string, error) { if infra == nil { @@ -98,3 +81,68 @@ func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (str return "", fmt.Errorf("unsupported storage provider: '%v'", *infra) } } + +// GetHostMountPathFromSecretRef returns the hostPath configured for the given store. +func GetHostMountPathFromSecretRef(ctx context.Context, client client.Client, logger logr.Logger, store *druidv1alpha1.StoreSpec, namespace string) (string, error) { + if store.SecretRef == nil { + logger.Info("secretRef is not defined for store, using default hostPath", "namespace", namespace) + return LocalProviderDefaultMountPath, nil + } + + secret := &corev1.Secret{} + if err := client.Get(ctx, utils.Key(namespace, store.SecretRef.Name), secret); err != nil { + return "", err + } + + hostPath, ok := secret.Data[EtcdBackupSecretHostPath] + if !ok { + return LocalProviderDefaultMountPath, nil + } + + return string(hostPath), nil +} + +// GetProviderEnvVars returns provider-specific environment variables for the given store. +func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { + if store == nil { + return nil, nil + } + + var envVars []corev1.EnvVar + + provider, err := StorageProviderFromInfraProvider(store.Provider) + if err != nil { + return nil, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") + } + + switch provider { + case S3: + envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) + + case ABS: + envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) + + case GCS: + envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.VolumeMountPathGCSBackupSecret))) + envVars = append(envVars, utils.GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) + + case Swift: + envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) + + case OSS: + envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) + + case ECS: + if store.SecretRef == nil { + return nil, fmt.Errorf("no secretRef could be configured for backup store of ECS") + } + envVars = append(envVars, utils.GetEnvVarFromSecret(common.EnvECSEndpoint, store.SecretRef.Name, "endpoint", false)) + envVars = append(envVars, utils.GetEnvVarFromSecret(common.EnvECSAccessKeyID, store.SecretRef.Name, "accessKeyID", false)) + envVars = append(envVars, utils.GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) + + case OCS: + envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) + } + + return envVars, nil +} diff --git a/internal/utils/store_test.go b/internal/store/store_test.go similarity index 96% rename from internal/utils/store_test.go rename to internal/store/store_test.go index 0010bec9c..dca2f7a2d 100644 --- a/internal/utils/store_test.go +++ b/internal/store/store_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package utils +package store import ( "context" @@ -10,7 +10,9 @@ import ( "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/go-logr/logr" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -85,7 +87,7 @@ func TestGetHostMountPathFromSecretRef(t *testing.T) { existingObjects = append(existingObjects, sec) } cl := testutils.CreateTestFakeClientForObjects(tc.getErr, nil, nil, nil, existingObjects, client.ObjectKey{Name: existingSecretName, Namespace: testutils.TestNamespace}) - secretName := IfConditionOr(tc.secretExists, existingSecretName, nonExistingSecretName) + secretName := utils.IfConditionOr(tc.secretExists, existingSecretName, nonExistingSecretName) storeSpec := createStoreSpec(tc.secretRefDefined, secretName, testutils.TestNamespace) actualHostPath, err := GetHostMountPathFromSecretRef(context.Background(), cl, logger, storeSpec, testutils.TestNamespace) if tc.expectedErr != nil { diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index 6b1d7f134..c2726cfb8 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -5,8 +5,6 @@ package utils import ( - "fmt" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" @@ -50,48 +48,7 @@ func GetEnvVarFromSecret(name, secretName, secretKey string, optional bool) core } } -// GetProviderEnvVars returns provider-specific environment variables for the given store -func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { - var envVars []corev1.EnvVar - - provider, err := StorageProviderFromInfraProvider(store.Provider) - if err != nil { - return nil, fmt.Errorf("storage provider is not recognized while fetching secrets from environment variable") - } - - switch provider { - case S3: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAWSApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) - - case ABS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) - - case GCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.VolumeMountPathGCSBackupSecret))) - envVars = append(envVars, GetEnvVarFromSecret(common.EnvGoogleStorageAPIEndpoint, store.SecretRef.Name, "storageAPIEndpoint", true)) - - case Swift: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenstackApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) - - case OSS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvAlicloudApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) - - case ECS: - if store.SecretRef == nil { - return nil, fmt.Errorf("no secretRef could be configured for backup store of ECS") - } - envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSEndpoint, store.SecretRef.Name, "endpoint", false)) - envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSAccessKeyID, store.SecretRef.Name, "accessKeyID", false)) - envVars = append(envVars, GetEnvVarFromSecret(common.EnvECSSecretAccessKey, store.SecretRef.Name, "secretAccessKey", false)) - - case OCS: - envVars = append(envVars, GetEnvVarFromValue(common.EnvOpenshiftApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret)) - } - - return envVars, nil -} - -// GetBackupRestoreContainerEnvVars returns backup-restore container environment variables for the given store +// GetBackupRestoreContainerEnvVars returns non-provider-specific environment variables for the backup-restore container. func GetBackupRestoreContainerEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { var envVars []corev1.EnvVar @@ -105,11 +62,5 @@ func GetBackupRestoreContainerEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1. storageContainer := pointer.StringDeref(store.Container, "") envVars = append(envVars, GetEnvVarFromValue(common.EnvStorageContainer, storageContainer)) - providerEnvVars, err := GetProviderEnvVars(store) - if err != nil { - return nil, err - } - envVars = append(envVars, providerEnvVars...) - return envVars, nil } diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index a8e1825c3..9fb89f491 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -9,13 +9,12 @@ import ( "fmt" "time" - brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" + + brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/gardener/pkg/utils/test/matchers" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +22,9 @@ import ( "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var _ = Describe("Etcd Backup", func() { @@ -56,7 +58,7 @@ var _ = Describe("Etcd Backup", func() { By("Purge snapstore") snapstoreProvider := provider.Storage.Provider - if snapstoreProvider == utils.Local { + if snapstoreProvider == druidstore.Local { purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) } else { diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index 4cc7eee71..9d34a8217 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -7,15 +7,16 @@ package e2e import ( "context" "fmt" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "time" - brtypes "github.com/gardener/etcd-backup-restore/pkg/types" - "github.com/gardener/etcd-druid/internal/utils" - batchv1 "k8s.io/api/batch/v1" - "k8s.io/apimachinery/pkg/types" + druidstore "github.com/gardener/etcd-druid/internal/store" + brtypes "github.com/gardener/etcd-backup-restore/pkg/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -50,7 +51,7 @@ var _ = Describe("Etcd Compaction", func() { By("Purge snapstore") snapstoreProvider := provider.Storage.Provider - if snapstoreProvider == utils.Local { + if snapstoreProvider == druidstore.Local { purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) } else { @@ -142,7 +143,7 @@ var _ = Describe("Etcd Compaction", func() { defer cancelFunc() req := types.NamespacedName{ - Name: etcd.GetCompactionJobName(), + Name: druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta), Namespace: etcd.Namespace, } diff --git a/test/e2e/etcd_multi_node_test.go b/test/e2e/etcd_multi_node_test.go index 911f73a09..4155dd142 100644 --- a/test/e2e/etcd_multi_node_test.go +++ b/test/e2e/etcd_multi_node_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -53,7 +53,7 @@ var _ = Describe("Etcd", func() { By("Purge snapstore") snapstoreProvider := provider.Storage.Provider - if snapstoreProvider == utils.Local { + if snapstoreProvider == druidstore.Local { purgeLocalSnapstoreJob := purgeLocalSnapstore(parentCtx, cl, storageContainer) defer cleanUpTestHelperJob(parentCtx, cl, purgeLocalSnapstoreJob.Name) } else { diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 1a1cf71cc..464928a97 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -16,7 +16,7 @@ import ( "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" "github.com/gardener/etcd-backup-restore/pkg/snapstore" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" @@ -343,7 +343,7 @@ func getProviders() ([]TestProvider, error) { Name: "aws", Suffix: "aws", Storage: &Storage{ - Provider: utils.S3, + Provider: druidstore.S3, SecretData: map[string][]byte{ "accessKeyID": []byte(s3AccessKeyID), "secretAccessKey": []byte(s3SecretAccessKey), @@ -365,7 +365,7 @@ func getProviders() ([]TestProvider, error) { Name: "az", Suffix: "az", Storage: &Storage{ - Provider: utils.ABS, + Provider: druidstore.ABS, SecretData: map[string][]byte{ "storageAccount": []byte(absStorageAccount), "storageKey": []byte(absStorageKey), @@ -385,7 +385,7 @@ func getProviders() ([]TestProvider, error) { Name: "gcp", Suffix: "gcp", Storage: &Storage{ - Provider: utils.GCS, + Provider: druidstore.GCS, SecretData: map[string][]byte{ "serviceaccount.json": gcsSA, }, @@ -397,7 +397,7 @@ func getProviders() ([]TestProvider, error) { Name: "local", Suffix: "local", Storage: &Storage{ - Provider: utils.Local, + Provider: druidstore.Local, SecretData: map[string][]byte{}, }, } diff --git a/test/integration/controllers/compaction/reconciler_test.go b/test/integration/controllers/compaction/reconciler_test.go index e5311f4b5..7c9a36378 100644 --- a/test/integration/controllers/compaction/reconciler_test.go +++ b/test/integration/controllers/compaction/reconciler_test.go @@ -11,8 +11,9 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/test/matchers" . "github.com/onsi/ginkgo/v2" @@ -278,7 +279,7 @@ var _ = Describe("Compaction Controller", func() { }) func validateEtcdForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1.Job) { - store, err := utils.StorageProviderFromInfraProvider(instance.Spec.Backup.Store.Provider) + store, err := druidstore.StorageProviderFromInfraProvider(instance.Spec.Backup.Store.Provider) Expect(err).NotTo(HaveOccurred()) Expect(*j).To(MatchFields(IgnoreExtras, Fields{ diff --git a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go index 66218b20b..067bb7205 100644 --- a/test/integration/controllers/etcdcopybackupstask/reconciler_test.go +++ b/test/integration/controllers/etcdcopybackupstask/reconciler_test.go @@ -11,7 +11,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" - "github.com/gardener/etcd-druid/internal/utils" + druidstore "github.com/gardener/etcd-druid/internal/store" testutils "github.com/gardener/etcd-druid/test/utils" "github.com/gardener/gardener/pkg/controllerutils" @@ -122,9 +122,9 @@ var _ = Describe("EtcdCopyBackupsTask Controller", func() { }) func matchJob(task *druidv1alpha1.EtcdCopyBackupsTask, imageVector imagevector.ImageVector) gomegatypes.GomegaMatcher { - sourceProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.SourceStore.Provider) + sourceProvider, err := druidstore.StorageProviderFromInfraProvider(task.Spec.SourceStore.Provider) Expect(err).NotTo(HaveOccurred()) - targetProvider, err := utils.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) + targetProvider, err := druidstore.StorageProviderFromInfraProvider(task.Spec.TargetStore.Provider) Expect(err).NotTo(HaveOccurred()) images, err := imagevector.FindImages(imageVector, []string{common.ImageKeyEtcdBackupRestore}) From d342ffcd4881a8d38242be17aa5c6f0e873def5c Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 24 May 2024 14:28:44 +0530 Subject: [PATCH 201/235] Update gardener/gardener to v1.86.4 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a6a7dce85..44c52b57b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/gardener/etcd-backup-restore v0.26.0 - github.com/gardener/gardener v1.86.0 + github.com/gardener/gardener v1.86.4 github.com/go-logr/logr v1.2.4 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index cf52f645f..391620d68 100644 --- a/go.sum +++ b/go.sum @@ -145,8 +145,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gardener/etcd-backup-restore v0.26.0 h1:ux0U5OT6IwtFiGbx4d3q+KYGRg6dTxBULPm8DL73E98= github.com/gardener/etcd-backup-restore v0.26.0/go.mod h1:cAY12PXK6oJgh5X/V6AuQ8eCpJspPNlnbNSwwjToj18= -github.com/gardener/gardener v1.86.0 h1:r0DjNZbUc2F/feWDrySZ/AU/5LAr0LoKXdraFtPb6NQ= -github.com/gardener/gardener v1.86.0/go.mod h1:8eHlXs2EkaghrgQwK8qEiVw3dZGpNJaq+I9IkPpReA4= +github.com/gardener/gardener v1.86.4 h1:uxMalw67jddMXliPhv5vs1nOB7G8hbFE1XKoyZjIMlE= +github.com/gardener/gardener v1.86.4/go.mod h1:8eHlXs2EkaghrgQwK8qEiVw3dZGpNJaq+I9IkPpReA4= github.com/gardener/hvpa-controller/api v0.5.0 h1:f4F3O7YUrenwh4S3TgPREPiB287JjjUiUL18OqPLyAA= github.com/gardener/hvpa-controller/api v0.5.0/go.mod h1:QQl3ELkCaki+8RhXl0FZMfvnm0WCGwGJlGmrxJj6lvM= github.com/gardener/machine-controller-manager v0.50.0 h1:3dcQjzueFU1TGgprV00adjb3OCR99myTBx8DQGxywks= From f78a88e9293bd4dfaf6a74fdfc12fc5d183e12b3 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 24 May 2024 19:29:11 +0530 Subject: [PATCH 202/235] Add missing finalizer to Etcd resource, fix it tests, fix generated CRDs --- Makefile | 1 + ...d.gardener.cloud_etcdcopybackupstasks.yaml | 2 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 82 +++++++++++++++++-- ...d.gardener.cloud_etcdcopybackupstasks.yaml | 2 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 82 +++++++++++++++++-- internal/controller/etcd/reconcile_delete.go | 5 +- internal/controller/etcd/reconcile_spec.go | 30 +++++-- test/it/controller/etcd/assertions.go | 5 +- test/it/controller/etcd/reconciler_test.go | 20 +++++ 9 files changed, 198 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index fbe984072..16b738880 100644 --- a/Makefile +++ b/Makefile @@ -162,6 +162,7 @@ deploy: $(SKAFFOLD) $(HELM) .PHONY: deploy-dev deploy-dev: $(SKAFFOLD) $(HELM) $(SKAFFOLD) dev -m etcd-druid --trigger='manual' + .PHONY: deploy-debug deploy-debug: $(SKAFFOLD) $(HELM) $(SKAFFOLD) debug -m etcd-druid diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 16951ef91..f395f5c97 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -154,7 +154,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index 8bb938ba7..3d0586e33 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -193,6 +193,12 @@ spec: leadership status of corresponding etcd is checked. type: string type: object + maxBackupsLimitBasedGC: + description: MaxBackupsLimitBasedGC defines the maximum number + of Full snapshots to retain in Limit Based GarbageCollectionPolicy + All full snapshots beyond this limit will be garbage collected. + format: int32 + type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. @@ -333,7 +339,8 @@ spec: type: object type: object etcd: - description: EtcdConfig defines parameters associated etcd deployed + description: EtcdConfig defines the configuration for the etcd cluster + to be deployed. properties: authSecretRef: description: SecretReference represents a Secret Reference. It @@ -1680,7 +1687,7 @@ spec: properties: autoCompactionMode: description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore + mode or 'revision' mode for etcd and embedded-etcd of backup-restore sidecar. enum: - periodic @@ -1688,7 +1695,7 @@ spec: type: string autoCompactionRetention: description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore + length for etcd as well as for embedded-etcd of backup-restore sidecar. type: string type: object @@ -1740,7 +1747,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: @@ -1828,12 +1835,73 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: LastError represents the last occurred error. + description: 'LastError represents the last occurred error. Deprecated: + Use LastErrors instead.' type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + description: LastError stores details of the most recent error encountered + for a resource. + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + observedAt: + description: ObservedAt is the time the error was observed. + format: date-time + type: string + required: + - code + - description + - observedAt + type: object + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. + properties: + description: + description: Description describes the last operation. + type: string + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was last updated. + format: date-time + type: string + runID: + description: RunID correlates an operation with a reconciliation + run. Every time an Etcd resource is reconciled (barring status + reconciliation which is periodic), a unique ID is generated + which can be used to correlate all actions done as part of a + single reconcile run. Capturing this as part of LastOperation + aids in establishing this correlation. This further helps in + also easily filtering reconcile logs as all structured logs + in a reconciliation run should have the `runID` referenced. + type: string + state: + description: State is the state of the last operation. + type: string + type: + description: Type is the type of last operation. + type: string + required: + - description + - lastUpdateTime + - runID + - state + - type + type: object members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about a etcd cluster + description: EtcdMemberStatus holds information about etcd cluster membership. properties: id: @@ -1883,7 +1951,7 @@ spec: format: int32 type: integer replicas: - description: Replicas is the replica count of the etcd resource. + description: Replicas is the replica count of the etcd cluster. format: int32 type: integer serviceName: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml index 16951ef91..f395f5c97 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml @@ -154,7 +154,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 8bb938ba7..3d0586e33 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -193,6 +193,12 @@ spec: leadership status of corresponding etcd is checked. type: string type: object + maxBackupsLimitBasedGC: + description: MaxBackupsLimitBasedGC defines the maximum number + of Full snapshots to retain in Limit Based GarbageCollectionPolicy + All full snapshots beyond this limit will be garbage collected. + format: int32 + type: integer port: description: Port define the port on which etcd-backup-restore server will be exposed. @@ -333,7 +339,8 @@ spec: type: object type: object etcd: - description: EtcdConfig defines parameters associated etcd deployed + description: EtcdConfig defines the configuration for the etcd cluster + to be deployed. properties: authSecretRef: description: SecretReference represents a Secret Reference. It @@ -1680,7 +1687,7 @@ spec: properties: autoCompactionMode: description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore + mode or 'revision' mode for etcd and embedded-etcd of backup-restore sidecar. enum: - periodic @@ -1688,7 +1695,7 @@ spec: type: string autoCompactionRetention: description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore + length for etcd as well as for embedded-etcd of backup-restore sidecar. type: string type: object @@ -1740,7 +1747,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about + description: A human-readable message indicating details about the transition. type: string reason: @@ -1828,12 +1835,73 @@ spec: type: object x-kubernetes-map-type: atomic lastError: - description: LastError represents the last occurred error. + description: 'LastError represents the last occurred error. Deprecated: + Use LastErrors instead.' type: string + lastErrors: + description: LastErrors captures errors that occurred during the last + operation. + items: + description: LastError stores details of the most recent error encountered + for a resource. + properties: + code: + description: Code is an error code that uniquely identifies + an error. + type: string + description: + description: Description is a human-readable message indicating + details of the error. + type: string + observedAt: + description: ObservedAt is the time the error was observed. + format: date-time + type: string + required: + - code + - description + - observedAt + type: object + type: array + lastOperation: + description: LastOperation indicates the last operation performed + on this resource. + properties: + description: + description: Description describes the last operation. + type: string + lastUpdateTime: + description: LastUpdateTime is the time at which the operation + was last updated. + format: date-time + type: string + runID: + description: RunID correlates an operation with a reconciliation + run. Every time an Etcd resource is reconciled (barring status + reconciliation which is periodic), a unique ID is generated + which can be used to correlate all actions done as part of a + single reconcile run. Capturing this as part of LastOperation + aids in establishing this correlation. This further helps in + also easily filtering reconcile logs as all structured logs + in a reconciliation run should have the `runID` referenced. + type: string + state: + description: State is the state of the last operation. + type: string + type: + description: Type is the type of last operation. + type: string + required: + - description + - lastUpdateTime + - runID + - state + - type + type: object members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about a etcd cluster + description: EtcdMemberStatus holds information about etcd cluster membership. properties: id: @@ -1883,7 +1951,7 @@ spec: format: int32 type: integer replicas: - description: Replicas is the replica count of the etcd resource. + description: Replicas is the replica count of the etcd cluster. format: int32 type: integer serviceName: diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 591629ac6..2a147cc88 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -13,6 +13,7 @@ import ( "github.com/gardener/etcd-druid/internal/component" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -85,10 +86,6 @@ func (r *Reconciler) removeFinalizer(ctx component.OperatorContext, etcdObjKey c if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { return result } - //etcd := &druidv1alpha1.Etcd{} - //if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { - // return result - //} ctx.Logger.Info("Removing finalizer", "finalizerName", common.FinalizerName) if err := controllerutils.RemoveFinalizers(ctx, r.client, etcdPartialObjMeta, common.FinalizerName); client.IgnoreNotFound(err) != nil { return ctrlutils.ReconcileWithError(err) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index f7b17a890..5228ca025 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -6,17 +6,22 @@ package etcd import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/component" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { reconcileStepFns := []reconcileFn{ r.recordReconcileStartOperation, + r.ensureFinalizer, r.syncEtcdResources, r.updateObservedGeneration, r.recordReconcileSuccessOperation, @@ -46,17 +51,24 @@ func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, et return ctrlutils.ReconcileWithError(err) } } - //if _, ok := etcd.Annotations[v1beta1constants.GardenerOperation]; ok { - // ctx.Logger.Info("Removing operation annotation") - // withOpAnnotation := etcd.DeepCopy() - // delete(etcd.Annotations, v1beta1constants.GardenerOperation) - // if err := r.client.Patch(ctx, etcd, client.MergeFrom(withOpAnnotation)); err != nil { - // ctx.Logger.Error(err, "failed to remove operation annotation") - // return ctrlutils.ReconcileWithError(err) - // } - //} return ctrlutils.ContinueReconcile() } + +func (r *Reconciler) ensureFinalizer(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata() + if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + if !controllerutil.ContainsFinalizer(etcdPartialObjMeta, common.FinalizerName) { + r.logger.Info("Adding finalizer", "finalizerName", common.FinalizerName) + if err := controllerutils.AddFinalizers(ctx, r.client, etcdPartialObjMeta, common.FinalizerName); err != nil { + ctx.Logger.Error(err, "failed to add finalizer") + return ctrlutils.ReconcileWithError(err) + } + } + return ctrlutils.ContinueReconcile() +} + func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { diff --git a/test/it/controller/etcd/assertions.go b/test/it/controller/etcd/assertions.go index 0539d261d..ed3fd62f6 100644 --- a/test/it/controller/etcd/assertions.go +++ b/test/it/controller/etcd/assertions.go @@ -312,9 +312,10 @@ func assertETCDFinalizer(t *testing.T, cl client.Client, etcdObjectKey client.Ob checkFn := func() error { etcdInstance := &druidv1alpha1.Etcd{} if err := cl.Get(context.Background(), etcdObjectKey, etcdInstance); err != nil { - if apierrors.IsNotFound(err) { + if apierrors.IsNotFound(err) && !expectedFinalizerPresent { // Once the finalizer is removed, the etcd resource will be removed by k8s very quickly. - // If we find that the resource was indeed removed then this check will pass. + // If we find that the resource was indeed removed, and if we also expect the finalizer + // to be absent, then this check will pass. return nil } return err diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index a681a03bb..3901ec5e3 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -67,6 +67,7 @@ func TestEtcdReconcileSpecWithNoAutoReconcile(t *testing.T) { name string fn func(t *testing.T, testNamespace string, reconcilerTestEnv ReconcilerTestEnv) }{ + {"should add finalizer to etcd when etcd resource is created", testAddFinalizerToEtcd}, {"should create all managed resources when etcd resource is created", testAllManagedResourcesAreCreated}, {"should succeed only in creation of some resources and not all and should record error in lastErrors and lastOperation", testFailureToCreateAllResources}, {"should not reconcile spec when reconciliation is suspended", testWhenReconciliationIsSuspended}, @@ -84,6 +85,25 @@ func TestEtcdReconcileSpecWithNoAutoReconcile(t *testing.T) { } } +func testAddFinalizerToEtcd(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { + const ( + timeout = time.Minute * 2 + pollingInterval = time.Second * 2 + ) + // ***************** setup ***************** + g := NewWithT(t) + etcdInstance := testutils.EtcdBuilderWithDefaults(testutils.TestEtcdName, testNs). + WithReplicas(3). + Build() + etcdInstance.Spec.Backup.Store = &druidv1alpha1.StoreSpec{} // empty store spec since backups are not required for this test + cl := reconcilerTestEnv.itTestEnv.GetClient() + ctx := context.Background() + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + // ***************** test etcd spec reconciliation ***************** + assertETCDFinalizer(t, cl, client.ObjectKeyFromObject(etcdInstance), true, timeout, pollingInterval) +} + func testAllManagedResourcesAreCreated(t *testing.T, testNs string, reconcilerTestEnv ReconcilerTestEnv) { const ( timeout = time.Minute * 2 From 5a9ecf9ff4abc0793c08017328694fe2abb14cc8 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 24 May 2024 23:04:50 +0530 Subject: [PATCH 203/235] Address review comments by @ashwani2k and @ishan16696 --- internal/component/snapshotlease/snapshotlease.go | 2 +- internal/controller/etcd/reconcile_delete.go | 2 +- internal/controller/etcd/reconcile_spec.go | 2 +- internal/utils/envvar.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/component/snapshotlease/snapshotlease.go b/internal/component/snapshotlease/snapshotlease.go index ab4207bf6..e658bf894 100644 --- a/internal/component/snapshotlease/snapshotlease.go +++ b/internal/component/snapshotlease/snapshotlease.go @@ -86,7 +86,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) return druiderr.WrapError(err, ErrSyncSnapshotLease, "Sync", - fmt.Sprintf("Failed to delete existing snapshot leases due to backup being disabled for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) + fmt.Sprintf("Failed to delete existing snapshot leases (due to backup being disabled for etcd) due to reason: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) }) } diff --git a/internal/controller/etcd/reconcile_delete.go b/internal/controller/etcd/reconcile_delete.go index 2a147cc88..d08b0bdfc 100644 --- a/internal/controller/etcd/reconcile_delete.go +++ b/internal/controller/etcd/reconcile_delete.go @@ -52,7 +52,7 @@ func (r *Reconciler) deleteEtcdResources(ctx component.OperatorContext, etcdObjK }, }) } - ctx.Logger.Info("triggering triggerDeletionFlow operators for all resources") + ctx.Logger.Info("triggering triggerDeletionFlow operators for all druid-managed resources") if errs := utils.RunConcurrently(ctx, deleteTasks); len(errs) > 0 { return ctrlutils.ReconcileWithError(errs...) } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 5228ca025..402b24e1a 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -60,7 +60,7 @@ func (r *Reconciler) ensureFinalizer(ctx component.OperatorContext, etcdObjKey c return result } if !controllerutil.ContainsFinalizer(etcdPartialObjMeta, common.FinalizerName) { - r.logger.Info("Adding finalizer", "finalizerName", common.FinalizerName) + ctx.Logger.Info("Adding finalizer", "finalizerName", common.FinalizerName) if err := controllerutils.AddFinalizers(ctx, r.client, etcdPartialObjMeta, common.FinalizerName); err != nil { ctx.Logger.Error(err, "failed to add finalizer") return ctrlutils.ReconcileWithError(err) diff --git a/internal/utils/envvar.go b/internal/utils/envvar.go index c2726cfb8..9487eadcf 100644 --- a/internal/utils/envvar.go +++ b/internal/utils/envvar.go @@ -20,8 +20,8 @@ func GetEnvVarFromValue(name, value string) corev1.EnvVar { } } -// GetEnvVarFromFieldPath returns environment variable object with provided name and value from field path -func GetEnvVarFromFieldPath(name, fieldPath string) corev1.EnvVar { +// getEnvVarFromFieldPath returns environment variable object with provided name and value from field path +func getEnvVarFromFieldPath(name, fieldPath string) corev1.EnvVar { return corev1.EnvVar{ Name: name, ValueFrom: &corev1.EnvVarSource{ @@ -52,8 +52,8 @@ func GetEnvVarFromSecret(name, secretName, secretKey string, optional bool) core func GetBackupRestoreContainerEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error) { var envVars []corev1.EnvVar - envVars = append(envVars, GetEnvVarFromFieldPath(common.EnvPodName, "metadata.name")) - envVars = append(envVars, GetEnvVarFromFieldPath(common.EnvPodNamespace, "metadata.namespace")) + envVars = append(envVars, getEnvVarFromFieldPath(common.EnvPodName, "metadata.name")) + envVars = append(envVars, getEnvVarFromFieldPath(common.EnvPodNamespace, "metadata.namespace")) if store == nil { return envVars, nil From 7dfac67522ff53bf1ccbbd6095f1dd7eb3a62375 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sun, 26 May 2024 13:56:09 +0530 Subject: [PATCH 204/235] addressed @ashwani2k comment, removed predicate package and moved the predicates to compaction controller package, removed ginkgo --- Makefile | 2 +- internal/controller/compaction/register.go | 81 ++++- .../controller/compaction/register_test.go | 289 +++++++++++++++++ internal/controller/predicate/predicate.go | 90 ------ .../predicate/predicate_suite_test.go | 17 - .../controller/predicate/predicate_test.go | 294 ------------------ 6 files changed, 368 insertions(+), 405 deletions(-) create mode 100644 internal/controller/compaction/register_test.go delete mode 100644 internal/controller/predicate/predicate.go delete mode 100644 internal/controller/predicate/predicate_suite_test.go delete mode 100644 internal/controller/predicate/predicate_test.go diff --git a/Makefile b/Makefile index 16b738880..3bb1fc1c3 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ test: $(GINKGO) $(GOTESTFMT) ./internal/mapper/... \ ./internal/metrics/... # run the golang native unit tests. - @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./api/... ./internal/controller/etcd/... ./internal/component/... ./internal/utils/... ./internal/webhook/... + @TEST_COV="true" "$(HACK_DIR)/test-go.sh" ./api/... ./internal/controller/etcd/... ./internal/controller/compaction/... ./internal/component/... ./internal/utils/... ./internal/webhook/... .PHONY: test-integration test-integration: $(GINKGO) $(SETUP_ENVTEST) $(GOTESTFMT) diff --git a/internal/controller/compaction/register.go b/internal/controller/compaction/register.go index f83e58bbd..faf006091 100644 --- a/internal/controller/compaction/register.go +++ b/internal/controller/compaction/register.go @@ -5,8 +5,13 @@ package compaction import ( + "reflect" + "strings" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - druidpredicates "github.com/gardener/etcd-druid/internal/controller/predicate" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" @@ -27,9 +32,79 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { }). For(&druidv1alpha1.Etcd{}). WithEventFilter(predicate. - Or(druidpredicates.SnapshotRevisionChanged(), - druidpredicates.JobStatusChanged())). + Or(snapshotRevisionChanged(), jobStatusChanged())). Owns(&coordinationv1.Lease{}). Owns(&batchv1.Job{}). Complete(r) } + +// snapshotRevisionChanged is a predicate that is `true` if the passed lease object is a snapshot lease and if the lease +// object's holderIdentity is updated. +func snapshotRevisionChanged() predicate.Predicate { + isSnapshotLease := func(obj client.Object) bool { + lease, ok := obj.(*coordinationv1.Lease) + if !ok { + return false + } + + return strings.HasSuffix(lease.Name, "full-snap") || strings.HasSuffix(lease.Name, "delta-snap") + } + + holderIdentityChange := func(objOld, objNew client.Object) bool { + leaseOld, ok := objOld.(*coordinationv1.Lease) + if !ok { + return false + } + leaseNew, ok := objNew.(*coordinationv1.Lease) + if !ok { + return false + } + + return !reflect.DeepEqual(leaseOld.Spec.HolderIdentity, leaseNew.Spec.HolderIdentity) + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return isSnapshotLease(event.Object) + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return isSnapshotLease(event.ObjectNew) && holderIdentityChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return false + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return false + }, + } +} + +// jobStatusChanged is a predicate that is `true` if the status of a job changes. +func jobStatusChanged() predicate.Predicate { + statusChange := func(objOld, objNew client.Object) bool { + jobOld, ok := objOld.(*batchv1.Job) + if !ok { + return false + } + jobNew, ok := objNew.(*batchv1.Job) + if !ok { + return false + } + return !apiequality.Semantic.DeepEqual(jobOld.Status, jobNew.Status) + } + + return predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return false + }, + UpdateFunc: func(event event.UpdateEvent) bool { + return statusChange(event.ObjectOld, event.ObjectNew) + }, + GenericFunc: func(event event.GenericEvent) bool { + return false + }, + DeleteFunc: func(event event.DeleteEvent) bool { + return false + }, + } +} diff --git a/internal/controller/compaction/register_test.go b/internal/controller/compaction/register_test.go new file mode 100644 index 000000000..1d64c578d --- /dev/null +++ b/internal/controller/compaction/register_test.go @@ -0,0 +1,289 @@ +package compaction + +import ( + "crypto/rand" + "math/big" + "strconv" + "testing" + "time" + + . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +func TestSnapshotRevisionChangedForCreateEvents(t *testing.T) { + tests := []struct { + name string + isObjectLease bool + objectName string + isHolderIdentitySet bool + shouldAllowCreateEvent bool + }{ + { + name: "object is not a lease object", + isObjectLease: false, + objectName: "not-a-lease", + shouldAllowCreateEvent: false, + }, + { + name: "object is a lease object, but not a snapshot lease", + isObjectLease: true, + objectName: "different-lease", + shouldAllowCreateEvent: false, + }, + { + name: "object is a new delta-snapshot lease, but holder identity is not set", + isObjectLease: true, + objectName: "etcd-test-delta-snap", + isHolderIdentitySet: false, + shouldAllowCreateEvent: true, + }, + { + name: "object is a new delta-snapshot lease, and holder identity is set", + isObjectLease: true, + objectName: "etcd-test-delta-snap", + isHolderIdentitySet: true, + shouldAllowCreateEvent: true, + }, + { + name: "object is a new full-snapshot lease, but holder identity is not set", + isObjectLease: true, + objectName: "etcd-test-full-snap", + isHolderIdentitySet: false, + shouldAllowCreateEvent: true, + }, + { + name: "object is a new full-snapshot lease, and holder identity is set", + isObjectLease: true, + objectName: "etcd-test-full-snap", + isHolderIdentitySet: true, + shouldAllowCreateEvent: true, + }, + } + + g := NewWithT(t) + t.Parallel() + predicate := snapshotRevisionChanged() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + obj, _ := createObjectsForSnapshotLeasePredicate(g, test.objectName, test.isObjectLease, true, test.isHolderIdentitySet, false) + g.Expect(predicate.Create(event.CreateEvent{Object: obj})).To(Equal(test.shouldAllowCreateEvent)) + }) + } +} + +func TestSnapshotRevisionChangedForUpdateEvents(t *testing.T) { + tests := []struct { + name string + isObjectLease bool + objectName string + isHolderIdentityChanged bool + shouldAllowUpdateEvent bool + }{ + { + name: "object is not a lease object", + isObjectLease: false, + objectName: "not-a-lease", + shouldAllowUpdateEvent: false, + }, + { + name: "object is a lease object, but not a snapshot lease", + isObjectLease: true, + objectName: "different-lease", + shouldAllowUpdateEvent: false, + }, + { + name: "object is a delta-snapshot lease, but holder identity is not changed", + isObjectLease: true, + objectName: "etcd-test-delta-snap", + isHolderIdentityChanged: false, + shouldAllowUpdateEvent: false, + }, + { + name: "object is a delta-snapshot lease, and holder identity is changed", + isObjectLease: true, + objectName: "etcd-test-delta-snap", + isHolderIdentityChanged: true, + shouldAllowUpdateEvent: true, + }, + { + name: "object is a full-snapshot lease, but holder identity is not changed", + isObjectLease: true, + objectName: "etcd-test-full-snap", + isHolderIdentityChanged: false, + shouldAllowUpdateEvent: false, + }, + { + name: "object is a full-snapshot lease, and holder identity is changed", + isObjectLease: true, + objectName: "etcd-test-full-snap", + isHolderIdentityChanged: true, + shouldAllowUpdateEvent: true, + }, + } + + g := NewWithT(t) + t.Parallel() + predicate := snapshotRevisionChanged() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + obj, oldObj := createObjectsForSnapshotLeasePredicate(g, test.objectName, test.isObjectLease, false, true, test.isHolderIdentityChanged) + g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: obj})).To(Equal(test.shouldAllowUpdateEvent)) + }) + } +} + +func TestSnapshotRevisionChangedForDeleteEvents(t *testing.T) { + g := NewWithT(t) + t.Parallel() + predicate := snapshotRevisionChanged() + obj, _ := createObjectsForSnapshotLeasePredicate(g, "etcd-test-delta-snap", true, true, true, true) + g.Expect(predicate.Delete(event.DeleteEvent{Object: obj})).To(BeFalse()) +} + +func TestSnapshotRevisionChangedForGenericEvents(t *testing.T) { + g := NewWithT(t) + t.Parallel() + predicate := snapshotRevisionChanged() + obj, _ := createObjectsForSnapshotLeasePredicate(g, "etcd-test-delta-snap", true, true, true, true) + g.Expect(predicate.Generic(event.GenericEvent{Object: obj})).To(BeFalse()) +} + +func TestJobStatusChangedForUpdateEvents(t *testing.T) { + tests := []struct { + name string + isObjectJob bool + isStatusChanged bool + shouldAllowUpdateEvent bool + }{ + { + name: "object is not a job", + isObjectJob: false, + shouldAllowUpdateEvent: false, + }, + { + name: "object is a job, but status is not changed", + isObjectJob: true, + isStatusChanged: false, + shouldAllowUpdateEvent: false, + }, + { + name: "object is a job, and status is changed", + isObjectJob: true, + isStatusChanged: true, + shouldAllowUpdateEvent: true, + }, + } + + g := NewWithT(t) + t.Parallel() + predicate := jobStatusChanged() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + obj, oldObj := createObjectsForJobStatusChangedPredicate(g, "etcd-test-compaction-job", test.isObjectJob, test.isStatusChanged) + g.Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: obj})).To(Equal(test.shouldAllowUpdateEvent)) + }) + } +} + +func createObjectsForJobStatusChangedPredicate(g *WithT, name string, isJobObj, isStatusChanged bool) (obj client.Object, oldObj client.Object) { + // if the object is not a job object, create a config map (random type chosen, could have been anything else as well). + if !isJobObj { + obj = createConfigMap(g, name) + oldObj = createConfigMap(g, name) + return + } + now := time.Now() + // create job objects + oldObj = &batchv1.Job{ + Status: batchv1.JobStatus{ + Active: 1, + StartTime: &metav1.Time{ + Time: now, + }, + }, + } + if isStatusChanged { + obj = &batchv1.Job{ + Status: batchv1.JobStatus{ + Succeeded: 1, + StartTime: &metav1.Time{ + Time: now, + }, + CompletionTime: &metav1.Time{ + Time: time.Now(), + }, + }, + } + } else { + obj = oldObj + } + return +} + +func createObjectsForSnapshotLeasePredicate(g *WithT, name string, isLeaseObj, isNewObject, isHolderIdentitySet, isHolderIdentityChanged bool) (obj client.Object, oldObj client.Object) { + // if the object is not a lease object, create a config map (random type chosen, could have been anything else as well). + if !isLeaseObj { + obj = createConfigMap(g, name) + oldObj = createConfigMap(g, name) + return + } + + // create lease objects + var holderIdentity, newHolderIdentity *string + // if it's a new object indicating a create event, create a new lease object and return. + if isNewObject { + if isHolderIdentitySet { + holderIdentity = pointer.String(strconv.Itoa(generateRandomInt(g))) + } + obj = createLease(name, holderIdentity) + return + } + + // create old and new lease objects. + holderIdentity = pointer.String(strconv.Itoa(generateRandomInt(g))) + oldObj = createLease(name, holderIdentity) + if isHolderIdentityChanged { + newHolderIdentity = pointer.String(strconv.Itoa(generateRandomInt(g))) + } else { + newHolderIdentity = holderIdentity + } + obj = createLease(name, newHolderIdentity) + + return +} + +func createLease(name string, holderIdentity *string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: holderIdentity, + }, + } +} + +func createConfigMap(g *WithT, name string) *corev1.ConfigMap { + randInt := generateRandomInt(g) + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: map[string]string{ + "k": strconv.Itoa(randInt), + }, + } +} + +func generateRandomInt(g *WithT) int { + randInt, err := rand.Int(rand.Reader, big.NewInt(1000)) + g.Expect(err).NotTo(HaveOccurred()) + return int(randInt.Int64()) +} diff --git a/internal/controller/predicate/predicate.go b/internal/controller/predicate/predicate.go deleted file mode 100644 index fe666e314..000000000 --- a/internal/controller/predicate/predicate.go +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -// Package predicate contains helper functions used for building predicates for -// event filtering for etcd-druid controllers. -package predicate - -import ( - "reflect" - "strings" - - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -// SnapshotRevisionChanged is a predicate that is `true` if the passed lease object is a snapshot lease and if the lease -// object's holderIdentity is updated. -func SnapshotRevisionChanged() predicate.Predicate { - isSnapshotLease := func(obj client.Object) bool { - lease, ok := obj.(*coordinationv1.Lease) - if !ok { - return false - } - - return strings.HasSuffix(lease.Name, "full-snap") || strings.HasSuffix(lease.Name, "delta-snap") - } - - holderIdentityChange := func(objOld, objNew client.Object) bool { - leaseOld, ok := objOld.(*coordinationv1.Lease) - if !ok { - return false - } - leaseNew, ok := objNew.(*coordinationv1.Lease) - if !ok { - return false - } - - return !reflect.DeepEqual(leaseOld.Spec.HolderIdentity, leaseNew.Spec.HolderIdentity) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return isSnapshotLease(event.Object) - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return isSnapshotLease(event.ObjectNew) && holderIdentityChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return false - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} - -// JobStatusChanged is a predicate that is `true` if the status of a job changes. -func JobStatusChanged() predicate.Predicate { - statusChange := func(objOld, objNew client.Object) bool { - jobOld, ok := objOld.(*batchv1.Job) - if !ok { - return false - } - jobNew, ok := objNew.(*batchv1.Job) - if !ok { - return false - } - return !apiequality.Semantic.DeepEqual(jobOld.Status, jobNew.Status) - } - - return predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return false - }, - UpdateFunc: func(event event.UpdateEvent) bool { - return statusChange(event.ObjectOld, event.ObjectNew) - }, - GenericFunc: func(event event.GenericEvent) bool { - return false - }, - DeleteFunc: func(event event.DeleteEvent) bool { - return false - }, - } -} diff --git a/internal/controller/predicate/predicate_suite_test.go b/internal/controller/predicate/predicate_suite_test.go deleted file mode 100644 index d800fe30e..000000000 --- a/internal/controller/predicate/predicate_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package predicate_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestPredicate(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Predicate Suite") -} diff --git a/internal/controller/predicate/predicate_test.go b/internal/controller/predicate/predicate_test.go deleted file mode 100644 index aa2e08f29..000000000 --- a/internal/controller/predicate/predicate_test.go +++ /dev/null @@ -1,294 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package predicate_test - -import ( - . "github.com/gardener/etcd-druid/internal/controller/predicate" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - batchv1 "k8s.io/api/batch/v1" - coordinationv1 "k8s.io/api/coordination/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -var _ = Describe("Druid Predicate", func() { - var ( - obj, oldObj client.Object - - createEvent event.CreateEvent - updateEvent event.UpdateEvent - deleteEvent event.DeleteEvent - genericEvent event.GenericEvent - ) - - JustBeforeEach(func() { - createEvent = event.CreateEvent{ - Object: obj, - } - updateEvent = event.UpdateEvent{ - ObjectOld: oldObj, - ObjectNew: obj, - } - deleteEvent = event.DeleteEvent{ - Object: obj, - } - genericEvent = event.GenericEvent{ - Object: obj, - } - }) - - Describe("#Lease", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = SnapshotRevisionChanged() - }) - - Context("when holder identity is nil for delta snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity matches for delta snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity differs for delta snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("5"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-delta-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity is nil for full snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity matches for full snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity differs for full snap leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("5"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-full-snap", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when holder identity differs for any other leases", func() { - BeforeEach(func() { - obj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("5"), - }, - } - oldObj = &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.String("0"), - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - }) - - Describe("#Job", func() { - var pred predicate.Predicate - - JustBeforeEach(func() { - pred = JobStatusChanged() - }) - - Context("when status matches", func() { - BeforeEach(func() { - now := metav1.Now() - obj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: &now, - }, - } - oldObj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: &now, - }, - } - }) - - It("should return false", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - - Context("when status differs", func() { - BeforeEach(func() { - now := metav1.Now() - obj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: nil, - }, - } - oldObj = &batchv1.Job{ - Status: batchv1.JobStatus{ - CompletionTime: &now, - }, - } - }) - - It("should return true", func() { - gomega.Expect(pred.Create(createEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Update(updateEvent)).To(gomega.BeTrue()) - gomega.Expect(pred.Delete(deleteEvent)).To(gomega.BeFalse()) - gomega.Expect(pred.Generic(genericEvent)).To(gomega.BeFalse()) - }) - }) - }) -}) From e3c8627552157cfcd652a990e86c02770e39f0fd Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 27 May 2024 13:18:12 +0530 Subject: [PATCH 205/235] Add missing license headers --- api/v1alpha1/helper.go | 4 ++++ api/v1alpha1/helper_test.go | 4 ++++ api/v1alpha1/store.go | 4 ++++ internal/controller/compaction/register_test.go | 4 ++++ internal/controller/config.go | 4 ++++ internal/controller/register.go | 4 ++++ internal/images/embed.go | 4 ++++ internal/webhook/config.go | 4 ++++ internal/webhook/register.go | 4 ++++ 9 files changed, 36 insertions(+) diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index 64b6a2e22..a290aab45 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package v1alpha1 import ( diff --git a/api/v1alpha1/helper_test.go b/api/v1alpha1/helper_test.go index afc7cd423..4efad097b 100644 --- a/api/v1alpha1/helper_test.go +++ b/api/v1alpha1/helper_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package v1alpha1 import ( diff --git a/api/v1alpha1/store.go b/api/v1alpha1/store.go index df7b188f3..727103357 100644 --- a/api/v1alpha1/store.go +++ b/api/v1alpha1/store.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package v1alpha1 import corev1 "k8s.io/api/core/v1" diff --git a/internal/controller/compaction/register_test.go b/internal/controller/compaction/register_test.go index 1d64c578d..7233088e3 100644 --- a/internal/controller/compaction/register_test.go +++ b/internal/controller/compaction/register_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package compaction import ( diff --git a/internal/controller/config.go b/internal/controller/config.go index 25a4ed2df..45ae5ae6f 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package controller import ( diff --git a/internal/controller/register.go b/internal/controller/register.go index 9ff297183..40689f8e3 100644 --- a/internal/controller/register.go +++ b/internal/controller/register.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package controller import ( diff --git a/internal/images/embed.go b/internal/images/embed.go index fa5d8cc58..0b465ac1b 100644 --- a/internal/images/embed.go +++ b/internal/images/embed.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package images import ( diff --git a/internal/webhook/config.go b/internal/webhook/config.go index 7cac774b9..78b0a24a0 100644 --- a/internal/webhook/config.go +++ b/internal/webhook/config.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package webhook import ( diff --git a/internal/webhook/register.go b/internal/webhook/register.go index 635e977f6..81d9f39c8 100644 --- a/internal/webhook/register.go +++ b/internal/webhook/register.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package webhook import ( From ad6a1187ef59de72325863b676bd9f17b245b7ec Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 29 May 2024 11:00:38 +0530 Subject: [PATCH 206/235] Move g/g hack script generate-crds.sh to druid --- Makefile | 2 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 216 +++++++++++++++++- .../10-crd-druid.gardener.cloud_etcds.yaml | 216 +++++++++++++++++- config/crd/bases/doc.go | 2 +- hack/generate-crds.sh | 69 ++++++ 5 files changed, 495 insertions(+), 10 deletions(-) create mode 100755 hack/generate-crds.sh diff --git a/Makefile b/Makefile index 3bb1fc1c3..1e368d525 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ check-generate: # Generate manifests e.g. CRD, RBAC etc. .PHONY: manifests manifests: $(VGOPATH) $(CONTROLLER_GEN) - @GARDENER_HACK_DIR=$(GARDENER_HACK_DIR) VGOPATH=$(VGOPATH) go generate ./config/crd/bases + @HACK_DIR=$(HACK_DIR) VGOPATH=$(VGOPATH) go generate ./config/crd/bases @find "$(REPO_ROOT)/config/crd/bases" -name "*.yaml" -exec cp '{}' "$(REPO_ROOT)/charts/druid/charts/crds/templates/" \; @controller-gen rbac:roleName=manager-role paths="./internal/controller/..." diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml index 3d0586e33..984cc4ac7 100644 --- a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml @@ -646,11 +646,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -684,11 +686,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -701,6 +705,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -752,11 +757,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -790,14 +797,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -832,7 +842,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list @@ -866,11 +877,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -883,6 +896,48 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key in (value)` to select the group of + existing pods which pods will be taken into + consideration for the incoming pod's pod (anti) + affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value + is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature + gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key notin (value)` to select the group + of existing pods which pods will be taken + into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist + in the incoming pod labels will be ignored. + The default value is empty. The same key is + forbidden to exist in both mismatchLabelKeys + and labelSelector. Also, mismatchLabelKeys + cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -924,11 +979,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -952,6 +1009,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -975,6 +1033,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -996,7 +1055,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1029,11 +1089,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1046,6 +1108,44 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key in (value)` to select + the group of existing pods which pods will be + taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. Also, + matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key notin (value)` to + select the group of existing pods which pods will + be taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both mismatchLabelKeys and labelSelector. Also, + mismatchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1086,11 +1186,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1113,6 +1215,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1126,6 +1229,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules @@ -1156,7 +1260,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list @@ -1190,11 +1295,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1207,6 +1314,48 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key in (value)` to select the group of + existing pods which pods will be taken into + consideration for the incoming pod's pod (anti) + affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value + is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature + gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key notin (value)` to select the group + of existing pods which pods will be taken + into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist + in the incoming pod labels will be ignored. + The default value is empty. The same key is + forbidden to exist in both mismatchLabelKeys + and labelSelector. Also, mismatchLabelKeys + cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1248,11 +1397,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1276,6 +1427,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -1299,6 +1451,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod @@ -1320,7 +1473,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1353,11 +1507,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1370,6 +1526,44 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key in (value)` to select + the group of existing pods which pods will be + taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. Also, + matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key notin (value)` to + select the group of existing pods which pods will + be taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both mismatchLabelKeys and labelSelector. Also, + mismatchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1410,11 +1604,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1437,6 +1633,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1450,6 +1647,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object topologySpreadConstraints: @@ -1492,11 +1690,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1665,11 +1865,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1818,11 +2020,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1969,4 +2173,8 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.labelSelector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas status: {} diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 3d0586e33..984cc4ac7 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -646,11 +646,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -684,11 +686,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -701,6 +705,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -752,11 +757,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -790,14 +797,17 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array + x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -832,7 +842,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list @@ -866,11 +877,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -883,6 +896,48 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key in (value)` to select the group of + existing pods which pods will be taken into + consideration for the incoming pod's pod (anti) + affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value + is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature + gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key notin (value)` to select the group + of existing pods which pods will be taken + into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist + in the incoming pod labels will be ignored. + The default value is empty. The same key is + forbidden to exist in both mismatchLabelKeys + and labelSelector. Also, mismatchLabelKeys + cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -924,11 +979,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -952,6 +1009,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -975,6 +1033,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -996,7 +1055,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1029,11 +1089,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1046,6 +1108,44 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key in (value)` to select + the group of existing pods which pods will be + taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. Also, + matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key notin (value)` to + select the group of existing pods which pods will + be taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both mismatchLabelKeys and labelSelector. Also, + mismatchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1086,11 +1186,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1113,6 +1215,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1126,6 +1229,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules @@ -1156,7 +1260,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list @@ -1190,11 +1295,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1207,6 +1314,48 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key in (value)` to select the group of + existing pods which pods will be taken into + consideration for the incoming pod's pod (anti) + affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value + is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature + gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod + label keys to select which pods will be taken + into consideration. The keys are used to lookup + values from the incoming pod labels, those + key-value labels are merged with `labelSelector` + as `key notin (value)` to select the group + of existing pods which pods will be taken + into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist + in the incoming pod labels will be ignored. + The default value is empty. The same key is + forbidden to exist in both mismatchLabelKeys + and labelSelector. Also, mismatchLabelKeys + cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1248,11 +1397,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1276,6 +1427,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -1299,6 +1451,7 @@ spec: - weight type: object type: array + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod @@ -1320,7 +1473,8 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. + in this case pods. If it's null, this PodAffinityTerm + matches with no Pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1353,11 +1507,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1370,6 +1526,44 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key in (value)` to select + the group of existing pods which pods will be + taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both matchLabelKeys and labelSelector. Also, + matchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: MismatchLabelKeys is a set of pod label + keys to select which pods will be taken into consideration. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are merged + with `labelSelector` as `key notin (value)` to + select the group of existing pods which pods will + be taken into consideration for the incoming pod's + pod (anti) affinity. Keys that don't exist in + the incoming pod labels will be ignored. The default + value is empty. The same key is forbidden to exist + in both mismatchLabelKeys and labelSelector. Also, + mismatchLabelKeys cannot be set when labelSelector + isn't set. This is an alpha field and requires + enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1410,11 +1604,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1437,6 +1633,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1450,6 +1647,7 @@ spec: - topologyKey type: object type: array + x-kubernetes-list-type: atomic type: object type: object topologySpreadConstraints: @@ -1492,11 +1690,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1665,11 +1865,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1818,11 +2020,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1969,4 +2173,8 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.labelSelector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas status: {} diff --git a/config/crd/bases/doc.go b/config/crd/bases/doc.go index fbff11c8c..170a667b3 100644 --- a/config/crd/bases/doc.go +++ b/config/crd/bases/doc.go @@ -2,6 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 -//go:generate sh -c "bash $GARDENER_HACK_DIR/generate-crds.sh -p 10-crd- druid.gardener.cloud" +//go:generate $HACK_DIR/generate-crds.sh -p 10-crd- package bases diff --git a/hack/generate-crds.sh b/hack/generate-crds.sh new file mode 100755 index 000000000..56b8b9b3e --- /dev/null +++ b/hack/generate-crds.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +CRD_FILENAME_PREFIX="" +REPO_ROOT="$(git rev-parse --show-toplevel)" + +function create_usage() { + usage=$(printf '%s\n' " + usage: $(basename $0) [Options] + Options: + -p, --prefix (Optional) Prefix for the CRD name + ") + echo "$usage" +} + +function check_prerequisites() { + if ! command -v controller-gen &>/dev/null; then + echo >&2 "controller-gen not available" + exit 1 + fi +} + +function parse_flags() { + while test $# -gt 0; do + case "$1" in + -p | --prefix) + shift + CRD_FILENAME_PREFIX="$1" + ;; + -h | --help) + shift + echo "${USAGE}" + exit 0 + ;; + *) + echo >&2 "Unknown flag: $1" + exit 1 + ;; + esac + shift + done +} + +function generate_crds() { + local output_dir="$(pwd)" + + # generate crds + controller-gen crd paths="github.com/gardener/etcd-druid/api/v1alpha1" output:crd:dir="${output_dir}" output:stdout + + # rename files adding prefix if any specified + find "${output_dir}" -maxdepth 1 -name "druid.gardener.cloud*" -type f -print0 | + while IFS= read -r -d '' crd_file; do + crd_out_file="${output_dir}/${CRD_FILENAME_PREFIX}$(basename "$crd_file")" + mv "${crd_file}" "${crd_out_file}" + done +} + +export GO111MODULE=off +echo "Generating CRDs for druid.gardener.cloud" +check_prerequisites +parse_flags "$@" +generate_crds \ No newline at end of file From eda8e1852229da073cda759a9639d3b1cbb932ff Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 29 May 2024 12:19:07 +0530 Subject: [PATCH 207/235] added a comment explaining the order of lastOp update and op annotation removal --- internal/controller/etcd/reconcile_spec.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 402b24e1a..0ff83c827 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -25,6 +25,12 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etc r.syncEtcdResources, r.updateObservedGeneration, r.recordReconcileSuccessOperation, + // Removing the operation annotation after last operation recording is counter-intuitive. + // If we reverse the order where we first remove the operation annotation and then record the last operation then + // in case the operation annotation removal succeeds but the last operation recording fails, then the control + // will never enter this flow again and the last operation will never be recorded. Reason is that in + // there is a predicate check done in reconciler.canReconcile prior to entering this flow. That check will no longer + // succeed. r.removeOperationAnnotation, } From 080454f5a067017bc5f9dd52dc70669af5671838 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 29 May 2024 12:25:24 +0530 Subject: [PATCH 208/235] changed the comment --- internal/controller/etcd/reconcile_spec.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 0ff83c827..ee3ae1b5d 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -25,12 +25,12 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etc r.syncEtcdResources, r.updateObservedGeneration, r.recordReconcileSuccessOperation, - // Removing the operation annotation after last operation recording is counter-intuitive. + // Removing the operation annotation after last operation recording seems counter-intuitive. // If we reverse the order where we first remove the operation annotation and then record the last operation then // in case the operation annotation removal succeeds but the last operation recording fails, then the control - // will never enter this flow again and the last operation will never be recorded. Reason is that in - // there is a predicate check done in reconciler.canReconcile prior to entering this flow. That check will no longer - // succeed. + // will never enter this flow again and the last operation will never be recorded. + // Reason: there is a predicate check done in `reconciler.canReconcile` prior to entering this flow. + // That check will no longer succeed once the reconcile operation annotation has been removed. r.removeOperationAnnotation, } From 30a5ed57c916d51c23c440807aa0b7de5907f515 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Wed, 29 May 2024 13:57:55 +0530 Subject: [PATCH 209/235] added unit test for concurrent.go utility --- internal/utils/concurrent_test.go | 103 ++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 internal/utils/concurrent_test.go diff --git a/internal/utils/concurrent_test.go b/internal/utils/concurrent_test.go new file mode 100644 index 000000000..8ca015fcf --- /dev/null +++ b/internal/utils/concurrent_test.go @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 +package utils + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/gardener/etcd-druid/internal/component" + "github.com/go-logr/logr" + . "github.com/onsi/gomega" +) + +func TestRunConcurrentlyWithAllSuccessfulTasks(t *testing.T) { + tasks := []OperatorTask{ + createSuccessfulTaskWithDelay("task-1", 5*time.Millisecond), + createSuccessfulTaskWithDelay("task-2", 15*time.Millisecond), + createSuccessfulTaskWithDelay("task-3", 10*time.Millisecond), + } + g := NewWithT(t) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), "1e7632c1-fd2d-4a35-a690-c53fa68e1442") + g.Expect(RunConcurrently(opCtx, tasks)).To(HaveLen(0)) +} + +func TestRunConcurrentlyWithOnePanickyTask(t *testing.T) { + tasks := []OperatorTask{ + createSuccessfulTaskWithDelay("task-1", 5*time.Millisecond), + createPanickyTaskWithDelay("panicky-task-2", 15*time.Millisecond), + createSuccessfulTaskWithDelay("task-3", 10*time.Millisecond), + } + g := NewWithT(t) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), "1e7632c1-fd2d-4a35-a690-c53fa68e1442") + g.Expect(RunConcurrently(opCtx, tasks)).To(HaveLen(1)) +} + +func TestRunConcurrentlyWithPanickyAndErringTasks(t *testing.T) { + tasks := []OperatorTask{ + createSuccessfulTaskWithDelay("task-1", 5*time.Millisecond), + createPanickyTaskWithDelay("panicky-task-2", 15*time.Millisecond), + createSuccessfulTaskWithDelay("task-3", 10*time.Millisecond), + createErringTaskWithDelay("erring-task-4", 50*time.Millisecond), + } + g := NewWithT(t) + opCtx := component.NewOperatorContext(context.Background(), logr.Discard(), "1e7632c1-fd2d-4a35-a690-c53fa68e1442") + g.Expect(RunConcurrently(opCtx, tasks)).To(HaveLen(2)) +} + +func createSuccessfulTaskWithDelay(name string, delay time.Duration) OperatorTask { + return OperatorTask{ + Name: name, + Fn: func(ctx component.OperatorContext) error { + tick := time.NewTicker(delay) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + return nil + } + } + }, + } +} + +func createPanickyTaskWithDelay(name string, delay time.Duration) OperatorTask { + return OperatorTask{ + Name: name, + Fn: func(ctx component.OperatorContext) error { + tick := time.NewTicker(delay) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + panic("i panicked") + } + } + }, + } +} + +func createErringTaskWithDelay(name string, delay time.Duration) OperatorTask { + return OperatorTask{ + Name: name, + Fn: func(ctx component.OperatorContext) error { + tick := time.NewTicker(delay) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + return errors.New("this task will never succeed") + } + } + }, + } +} From 616d7bec5a4b21410b3e7b06819aea96fe2a0615 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 29 May 2024 15:25:37 +0530 Subject: [PATCH 210/235] Remove 10- prefix from CRD names --- ...-druid.gardener.cloud_etcdcopybackupstasks.yaml} | 0 ...cds.yaml => crd-druid.gardener.cloud_etcds.yaml} | 0 ...-druid.gardener.cloud_etcdcopybackupstasks.yaml} | 0 ...cds.yaml => crd-druid.gardener.cloud_etcds.yaml} | 0 config/crd/bases/doc.go | 2 +- config/crd/kustomization.yaml | 4 ++-- hack/generate-crds.sh | 13 +++++++------ test/integration/controllers/assets/assets.go | 4 ++-- test/it/controller/assets/assets.go | 4 ++-- 9 files changed, 14 insertions(+), 13 deletions(-) rename charts/druid/charts/crds/templates/{10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml => crd-druid.gardener.cloud_etcdcopybackupstasks.yaml} (100%) rename charts/druid/charts/crds/templates/{10-crd-druid.gardener.cloud_etcds.yaml => crd-druid.gardener.cloud_etcds.yaml} (100%) rename config/crd/bases/{10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml => crd-druid.gardener.cloud_etcdcopybackupstasks.yaml} (100%) rename config/crd/bases/{10-crd-druid.gardener.cloud_etcds.yaml => crd-druid.gardener.cloud_etcds.yaml} (100%) diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcdcopybackupstasks.yaml similarity index 100% rename from charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml rename to charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcdcopybackupstasks.yaml diff --git a/charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml similarity index 100% rename from charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml rename to charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml b/config/crd/bases/crd-druid.gardener.cloud_etcdcopybackupstasks.yaml similarity index 100% rename from config/crd/bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml rename to config/crd/bases/crd-druid.gardener.cloud_etcdcopybackupstasks.yaml diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml similarity index 100% rename from config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml rename to config/crd/bases/crd-druid.gardener.cloud_etcds.yaml diff --git a/config/crd/bases/doc.go b/config/crd/bases/doc.go index 170a667b3..efb1a4a48 100644 --- a/config/crd/bases/doc.go +++ b/config/crd/bases/doc.go @@ -2,6 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 -//go:generate $HACK_DIR/generate-crds.sh -p 10-crd- +//go:generate $HACK_DIR/generate-crds.sh -p crd- package bases diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 787f15087..cb0d6e28c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,8 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: - - bases/10-crd-druid.gardener.cloud_etcds.yaml - - bases/10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml + - bases/crd-druid.gardener.cloud_etcds.yaml + - bases/crd-druid.gardener.cloud_etcdcopybackupstasks.yaml # +kubebuilder:scaffold:crdkustomizeresource # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/hack/generate-crds.sh b/hack/generate-crds.sh index 56b8b9b3e..37232b558 100755 --- a/hack/generate-crds.sh +++ b/hack/generate-crds.sh @@ -9,7 +9,6 @@ set -o nounset set -o pipefail CRD_FILENAME_PREFIX="" -REPO_ROOT="$(git rev-parse --show-toplevel)" function create_usage() { usage=$(printf '%s\n' " @@ -55,11 +54,13 @@ function generate_crds() { controller-gen crd paths="github.com/gardener/etcd-druid/api/v1alpha1" output:crd:dir="${output_dir}" output:stdout # rename files adding prefix if any specified - find "${output_dir}" -maxdepth 1 -name "druid.gardener.cloud*" -type f -print0 | - while IFS= read -r -d '' crd_file; do - crd_out_file="${output_dir}/${CRD_FILENAME_PREFIX}$(basename "$crd_file")" - mv "${crd_file}" "${crd_out_file}" - done + if [ -n "${CRD_FILENAME_PREFIX}" ]; then + find "${output_dir}" -maxdepth 1 -name "druid.gardener.cloud*" -type f -print0 | + while IFS= read -r -d '' crd_file; do + crd_out_file="${output_dir}/${CRD_FILENAME_PREFIX}$(basename "$crd_file")" + mv "${crd_file}" "${crd_out_file}" + done + fi } export GO111MODULE=off diff --git a/test/integration/controllers/assets/assets.go b/test/integration/controllers/assets/assets.go index 2754c9194..e75ba6c95 100644 --- a/test/integration/controllers/assets/assets.go +++ b/test/integration/controllers/assets/assets.go @@ -15,12 +15,12 @@ import ( // GetEtcdCrdPath returns the path to the Etcd CRD. func GetEtcdCrdPath() string { - return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "10-crd-druid.gardener.cloud_etcds.yaml") + return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "crd-druid.gardener.cloud_etcds.yaml") } // GetEtcdCopyBackupsTaskCrdPath returns the path to the EtcdCopyBackupsTask CRD. func GetEtcdCopyBackupsTaskCrdPath() string { - return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml") + return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "crd-druid.gardener.cloud_etcdcopybackupstasks.yaml") } // CreateImageVector creates an image vector. diff --git a/test/it/controller/assets/assets.go b/test/it/controller/assets/assets.go index d3493de0d..21c005683 100644 --- a/test/it/controller/assets/assets.go +++ b/test/it/controller/assets/assets.go @@ -15,12 +15,12 @@ import ( // GetEtcdCrdPath returns the path to the Etcd CRD. func GetEtcdCrdPath() string { - return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "10-crd-druid.gardener.cloud_etcds.yaml") + return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "crd-druid.gardener.cloud_etcds.yaml") } // GetEtcdCopyBackupsTaskCrdPath returns the path to the EtcdCopyBackupsTask CRD. func GetEtcdCopyBackupsTaskCrdPath() string { - return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "10-crd-druid.gardener.cloud_etcdcopybackupstasks.yaml") + return filepath.Join("..", "..", "..", "..", "config", "crd", "bases", "crd-druid.gardener.cloud_etcdcopybackupstasks.yaml") } // GetEtcdCopyBackupsBaseChartPath returns the path to the etcd-copy-backups chart. From 16ea743b59ed6bebf69164f5739580189cf8345e Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 29 May 2024 15:52:49 +0530 Subject: [PATCH 211/235] Add missing vgopath setup to generate-crds script --- .../crd-druid.gardener.cloud_etcds.yaml | 212 +----------------- .../bases/crd-druid.gardener.cloud_etcds.yaml | 212 +----------------- hack/generate-crds.sh | 3 + hack/vgopath-setup.sh | 33 +++ 4 files changed, 44 insertions(+), 416 deletions(-) create mode 100755 hack/vgopath-setup.sh diff --git a/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml index 984cc4ac7..cb3524e17 100644 --- a/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml @@ -646,13 +646,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -686,13 +684,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -705,7 +701,6 @@ spec: - weight type: object type: array - x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -757,13 +752,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -797,17 +790,14 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array - x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -842,8 +832,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list @@ -877,13 +866,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -896,48 +883,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key in (value)` to select the group of - existing pods which pods will be taken into - consideration for the incoming pod's pod (anti) - affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value - is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature - gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key notin (value)` to select the group - of existing pods which pods will be taken - into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist - in the incoming pod labels will be ignored. - The default value is empty. The same key is - forbidden to exist in both mismatchLabelKeys - and labelSelector. Also, mismatchLabelKeys - cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling - MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -979,13 +924,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1009,7 +952,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -1033,7 +975,6 @@ spec: - weight type: object type: array - x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -1055,8 +996,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1089,13 +1029,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1108,44 +1046,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key in (value)` to select - the group of existing pods which pods will be - taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. Also, - matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key notin (value)` to - select the group of existing pods which pods will - be taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both mismatchLabelKeys and labelSelector. Also, - mismatchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1186,13 +1086,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1215,7 +1113,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1229,7 +1126,6 @@ spec: - topologyKey type: object type: array - x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules @@ -1260,8 +1156,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list @@ -1295,13 +1190,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1314,48 +1207,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key in (value)` to select the group of - existing pods which pods will be taken into - consideration for the incoming pod's pod (anti) - affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value - is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature - gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key notin (value)` to select the group - of existing pods which pods will be taken - into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist - in the incoming pod labels will be ignored. - The default value is empty. The same key is - forbidden to exist in both mismatchLabelKeys - and labelSelector. Also, mismatchLabelKeys - cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling - MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1397,13 +1248,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1427,7 +1276,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -1451,7 +1299,6 @@ spec: - weight type: object type: array - x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod @@ -1473,8 +1320,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1507,13 +1353,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1526,44 +1370,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key in (value)` to select - the group of existing pods which pods will be - taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. Also, - matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key notin (value)` to - select the group of existing pods which pods will - be taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both mismatchLabelKeys and labelSelector. Also, - mismatchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1604,13 +1410,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1633,7 +1437,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1647,7 +1450,6 @@ spec: - topologyKey type: object type: array - x-kubernetes-list-type: atomic type: object type: object topologySpreadConstraints: @@ -1690,13 +1492,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1865,13 +1665,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2020,13 +1818,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string diff --git a/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml index 984cc4ac7..cb3524e17 100644 --- a/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml @@ -646,13 +646,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -686,13 +684,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic weight: @@ -705,7 +701,6 @@ spec: - weight type: object type: array - x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -757,13 +752,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. @@ -797,17 +790,14 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic type: array - x-kubernetes-list-type: atomic required: - nodeSelectorTerms type: object @@ -842,8 +832,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list @@ -877,13 +866,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -896,48 +883,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key in (value)` to select the group of - existing pods which pods will be taken into - consideration for the incoming pod's pod (anti) - affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value - is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature - gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key notin (value)` to select the group - of existing pods which pods will be taken - into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist - in the incoming pod labels will be ignored. - The default value is empty. The same key is - forbidden to exist in both mismatchLabelKeys - and labelSelector. Also, mismatchLabelKeys - cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling - MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -979,13 +924,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1009,7 +952,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -1033,7 +975,6 @@ spec: - weight type: object type: array - x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will @@ -1055,8 +996,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1089,13 +1029,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1108,44 +1046,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key in (value)` to select - the group of existing pods which pods will be - taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. Also, - matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key notin (value)` to - select the group of existing pods which pods will - be taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both mismatchLabelKeys and labelSelector. Also, - mismatchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1186,13 +1086,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1215,7 +1113,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1229,7 +1126,6 @@ spec: - topologyKey type: object type: array - x-kubernetes-list-type: atomic type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules @@ -1260,8 +1156,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list @@ -1295,13 +1190,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1314,48 +1207,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key in (value)` to select the group of - existing pods which pods will be taken into - consideration for the incoming pod's pod (anti) - affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value - is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature - gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod - label keys to select which pods will be taken - into consideration. The keys are used to lookup - values from the incoming pod labels, those - key-value labels are merged with `labelSelector` - as `key notin (value)` to select the group - of existing pods which pods will be taken - into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist - in the incoming pod labels will be ignored. - The default value is empty. The same key is - forbidden to exist in both mismatchLabelKeys - and labelSelector. Also, mismatchLabelKeys - cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling - MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1397,13 +1248,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1427,7 +1276,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the @@ -1451,7 +1299,6 @@ spec: - weight type: object type: array - x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod @@ -1473,8 +1320,7 @@ spec: properties: labelSelector: description: A label query over a set of resources, - in this case pods. If it's null, this PodAffinityTerm - matches with no Pods. + in this case pods. properties: matchExpressions: description: matchExpressions is a list of label @@ -1507,13 +1353,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1526,44 +1370,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - matchLabelKeys: - description: MatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key in (value)` to select - the group of existing pods which pods will be - taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both matchLabelKeys and labelSelector. Also, - matchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: MismatchLabelKeys is a set of pod label - keys to select which pods will be taken into consideration. - The keys are used to lookup values from the incoming - pod labels, those key-value labels are merged - with `labelSelector` as `key notin (value)` to - select the group of existing pods which pods will - be taken into consideration for the incoming pod's - pod (anti) affinity. Keys that don't exist in - the incoming pod labels will be ignored. The default - value is empty. The same key is forbidden to exist - in both mismatchLabelKeys and labelSelector. Also, - mismatchLabelKeys cannot be set when labelSelector - isn't set. This is an alpha field and requires - enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic namespaceSelector: description: A label query over the set of namespaces that the term applies to. The term is applied @@ -1604,13 +1410,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1633,7 +1437,6 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods @@ -1647,7 +1450,6 @@ spec: - topologyKey type: object type: array - x-kubernetes-list-type: atomic type: object type: object topologySpreadConstraints: @@ -1690,13 +1492,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -1865,13 +1665,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -2020,13 +1818,11 @@ spec: items: type: string type: array - x-kubernetes-list-type: atomic required: - key - operator type: object type: array - x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string diff --git a/hack/generate-crds.sh b/hack/generate-crds.sh index 37232b558..7226c2c0e 100755 --- a/hack/generate-crds.sh +++ b/hack/generate-crds.sh @@ -63,6 +63,9 @@ function generate_crds() { fi } +# setup virtual GOPATH +source $(dirname $0)/vgopath-setup.sh + export GO111MODULE=off echo "Generating CRDs for druid.gardener.cloud" check_prerequisites diff --git a/hack/vgopath-setup.sh b/hack/vgopath-setup.sh new file mode 100755 index 000000000..ee841f385 --- /dev/null +++ b/hack/vgopath-setup.sh @@ -0,0 +1,33 @@ +# Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Ensure that if GOPATH is set, the GOPATH/{bin,pkg} directory exists. This seems to be not always +# the case in certain environments like Prow. As we will create a symlink against the bin folder we +# need to make sure that the bin directory is present in the GOPATH. +if [ -n "$GOPATH" ] && [ ! -d "$GOPATH/bin" ]; then mkdir -p "$GOPATH/bin"; fi +if [ -n "$GOPATH" ] && [ ! -d "$GOPATH/pkg" ]; then mkdir -p "$GOPATH/pkg"; fi + +VIRTUAL_GOPATH="$(mktemp -d)" +trap 'rm -rf "$VIRTUAL_GOPATH"' EXIT + +# Use REPO_ROOT if set, otherwise default to $SCRIPT_DIR/.. +TARGET_DIR="${REPO_ROOT:-$SCRIPT_DIR/..}" + +# Setup virtual GOPATH +(cd "$TARGET_DIR"; go mod download && "$VGOPATH" -o "$VIRTUAL_GOPATH") + +export GOROOT="${GOROOT:-"$(go env GOROOT)"}" +export GOPATH="$VIRTUAL_GOPATH" From 3f84ea274ec854d62fbd8948cb049626bbac7e9e Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Wed, 29 May 2024 17:03:27 +0530 Subject: [PATCH 212/235] Address review comments by @aaronfern, add missing license headers --- .ci/hack/doc.go | 16 +++------------- .ci/hack/prepare_release | 15 ++------------- .ci/hack/set_dependency_version | 15 ++------------- hack/vgopath-setup.sh | 14 ++------------ internal/controller/compaction/config.go | 17 +++++++++++------ 5 files changed, 20 insertions(+), 57 deletions(-) diff --git a/.ci/hack/doc.go b/.ci/hack/doc.go index fd4efa871..705812d95 100755 --- a/.ci/hack/doc.go +++ b/.ci/hack/doc.go @@ -1,16 +1,6 @@ -// Copyright 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 -// This package imports CI related scripts - it is to force `go mod` to see them as dependencies. +// Package ci imports CI related scripts - it is to force `go mod` to see them as dependencies. package ci diff --git a/.ci/hack/prepare_release b/.ci/hack/prepare_release index 42eeb6f1b..7e2620554 100755 --- a/.ci/hack/prepare_release +++ b/.ci/hack/prepare_release @@ -1,18 +1,7 @@ #!/usr/bin/env sh +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors # -# Copyright 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 set -e diff --git a/.ci/hack/set_dependency_version b/.ci/hack/set_dependency_version index 95997760f..19c719a87 100755 --- a/.ci/hack/set_dependency_version +++ b/.ci/hack/set_dependency_version @@ -1,18 +1,7 @@ #!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors # -# Copyright 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 import json import pathlib diff --git a/hack/vgopath-setup.sh b/hack/vgopath-setup.sh index ee841f385..4fea57650 100755 --- a/hack/vgopath-setup.sh +++ b/hack/vgopath-setup.sh @@ -1,16 +1,6 @@ -# Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/internal/controller/compaction/config.go b/internal/controller/compaction/config.go index 0ad891cb5..ae77cc231 100644 --- a/internal/controller/compaction/config.go +++ b/internal/controller/compaction/config.go @@ -66,13 +66,18 @@ func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { // Validate validates the config. func (cfg *Config) Validate() error { - if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, defaultMinWorkers, cfg.Workers); err != nil { - return err + if cfg.EnableBackupCompaction { + if err := utils.MustBeGreaterThanOrEqualTo(workersFlagName, defaultMinWorkers, cfg.Workers); err != nil { + return err + } + if err := utils.MustBeGreaterThan(eventsThresholdFlagName, 0, cfg.EventsThreshold); err != nil { + return err + } + if err := utils.MustBeGreaterThan(activeDeadlineDurationFlagName, 0, cfg.ActiveDeadlineDuration.Seconds()); err != nil { + return err + } } - if err := utils.MustBeGreaterThan(eventsThresholdFlagName, 0, cfg.EventsThreshold); err != nil { - return err - } - return utils.MustBeGreaterThan(activeDeadlineDurationFlagName, 0, cfg.ActiveDeadlineDuration.Seconds()) + return nil } // CaptureFeatureActivations captures all feature gates required by the controller into controller config From 41fb0096497160bbf4e72e1ee225f9089168d3f4 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 30 May 2024 09:04:31 +0530 Subject: [PATCH 213/235] Address review comment by @ishan16696 --- internal/common/constants.go | 6 ++---- internal/controller/compaction/reconciler.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/common/constants.go b/internal/common/constants.go index 188839bc2..bd0915330 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -5,8 +5,6 @@ package common const ( - // DefaultImageVectorFilePath is the path to the default image vector file. - DefaultImageVectorFilePath = "charts/images.yaml" // FinalizerName is the name of the etcd finalizer. FinalizerName = "druid.gardener.cloud/etcd-druid" // CheckSumKeyConfigMap is the key that is set by a configmap component and used by StatefulSet component to @@ -107,8 +105,8 @@ const ( ComponentNameServiceAccount = "etcd-service-account" // ComponentNameStatefulSet is the component name for statefulset resource. ComponentNameStatefulSet = "etcd-statefulset" - // ComponentNameCompactionJob is the component name for compaction job resource. - ComponentNameCompactionJob = "etcd-compaction-job" + // ComponentNameSnapshotCompactionJob is the component name for snapshot compaction job resource. + ComponentNameSnapshotCompactionJob = "etcd-snapshot-compaction-job" // ComponentNameEtcdCopyBackupsJob is the component name for copy-backup task resource. ComponentNameEtcdCopyBackupsJob = "etcd-copy-backups-job" ) diff --git a/internal/controller/compaction/reconciler.go b/internal/controller/compaction/reconciler.go index f6f6ea86c..a9fda276e 100644 --- a/internal/controller/compaction/reconciler.go +++ b/internal/controller/compaction/reconciler.go @@ -354,7 +354,7 @@ func (r *Reconciler) createCompactionJob(ctx context.Context, logger logr.Logger func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { jobLabels := map[string]string{ druidv1alpha1.LabelAppNameKey: druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta), - druidv1alpha1.LabelComponentKey: common.ComponentNameCompactionJob, + druidv1alpha1.LabelComponentKey: common.ComponentNameSnapshotCompactionJob, "networking.gardener.cloud/to-dns": "allowed", "networking.gardener.cloud/to-private-networks": "allowed", "networking.gardener.cloud/to-public-networks": "allowed", From 9ca05f252426a8986a1d40b5e367ea4b81f23054 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Thu, 30 May 2024 13:12:34 +0530 Subject: [PATCH 214/235] Address review comment by @aaronfern --- internal/common/constants.go | 3 +++ internal/component/configmap/configmap.go | 4 +--- internal/component/configmap/configmap_test.go | 2 +- internal/component/statefulset/builder.go | 4 ++-- internal/component/statefulset/constants.go | 1 - internal/component/statefulset/stsmatcher.go | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/common/constants.go b/internal/common/constants.go index bd0915330..fb91f5e8d 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -138,6 +138,9 @@ const ( VolumeNameProviderBackupSecret = "etcd-backup-secret" ) +// EtcdConfigFileName is the name of the etcd configuration file. +const EtcdConfigFileName = "etcd.conf.yaml" + // ModeOwnerReadWriteGroupRead is the file permissions used for volumes const ModeOwnerReadWriteGroupRead int32 = 0640 diff --git a/internal/component/configmap/configmap.go b/internal/component/configmap/configmap.go index cf1928503..c3f4e3c52 100644 --- a/internal/component/configmap/configmap.go +++ b/internal/component/configmap/configmap.go @@ -31,8 +31,6 @@ const ( ErrDeleteConfigMap druidv1alpha1.ErrorCode = "ERR_DELETE_CONFIGMAP" ) -const etcdConfigKey = "etcd.conf.yaml" - type _resource struct { client client.Client } @@ -119,7 +117,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, cm *corev1.ConfigMap) error { cm.Namespace = etcd.Namespace cm.Labels = getLabels(etcd) cm.OwnerReferences = []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(etcd.ObjectMeta)} - cm.Data = map[string]string{etcdConfigKey: string(cfgYaml)} + cm.Data = map[string]string{common.EtcdConfigFileName: string(cfgYaml)} return nil } diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 9fbea681f..4d8a964e6 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -330,7 +330,7 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C }), })) // Validate the etcd config data - actualETCDConfigYAML := actualConfigMap.Data[etcdConfigKey] + actualETCDConfigYAML := actualConfigMap.Data[common.EtcdConfigFileName] actualETCDConfig := make(map[string]interface{}) err := yaml.Unmarshal([]byte(actualETCDConfigYAML), &actualETCDConfig) g.Expect(err).ToNot(HaveOccurred()) diff --git a/internal/component/statefulset/builder.go b/internal/component/statefulset/builder.go index 07f4a9448..d1eca00a8 100644 --- a/internal/component/statefulset/builder.go +++ b/internal/component/statefulset/builder.go @@ -704,8 +704,8 @@ func (b *stsBuilder) getPodVolumes(ctx component.OperatorContext) ([]corev1.Volu }, Items: []corev1.KeyToPath{ { - Key: etcdConfigFileName, // etcd.conf.yaml key in the configmap, which contains the config for the etcd as yaml - Path: etcdConfigFileName, // sub-path under the volume mount path, to store the contents of configmap key etcd.conf.yaml + Key: common.EtcdConfigFileName, // etcd.conf.yaml key in the configmap, which contains the config for the etcd as yaml + Path: common.EtcdConfigFileName, // sub-path under the volume mount path, to store the contents of configmap key etcd.conf.yaml }, }, DefaultMode: pointer.Int32(common.ModeOwnerReadWriteGroupRead), diff --git a/internal/component/statefulset/constants.go b/internal/component/statefulset/constants.go index 6a423421f..5fb1f23ba 100644 --- a/internal/component/statefulset/constants.go +++ b/internal/component/statefulset/constants.go @@ -5,7 +5,6 @@ package statefulset const ( - etcdConfigFileName = "etcd.conf.yaml" etcdConfigFileMountPath = "/var/etcd/config/" ) diff --git a/internal/component/statefulset/stsmatcher.go b/internal/component/statefulset/stsmatcher.go index fcca8b275..f0e77865a 100644 --- a/internal/component/statefulset/stsmatcher.go +++ b/internal/component/statefulset/stsmatcher.go @@ -420,8 +420,8 @@ func (s StatefulSetMatcher) matchPodVolumes() gomegatypes.GomegaMatcher { "Name": Equal(druidv1alpha1.GetConfigMapName(s.etcd.ObjectMeta)), }), "Items": HaveExactElements(MatchFields(IgnoreExtras|IgnoreMissing, Fields{ - "Key": Equal(etcdConfigFileName), - "Path": Equal(etcdConfigFileName), + "Key": Equal(common.EtcdConfigFileName), + "Path": Equal(common.EtcdConfigFileName), })), "DefaultMode": PointTo(Equal(common.ModeOwnerReadWriteGroupRead)), })), From cdaa4942f082ba89f5576c925297e632d0129dbc Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 31 May 2024 14:22:14 +0530 Subject: [PATCH 215/235] Introduce `PreSync` for components to fix upgrade of etcds that were created by an older druid verison --- .../component/clientservice/clientservice.go | 3 + internal/component/configmap/configmap.go | 3 + internal/component/memberlease/memberlease.go | 3 + internal/component/peerservice/peerservice.go | 3 + .../poddisruptionbudget.go | 3 + internal/component/role/role.go | 3 + internal/component/rolebinding/rolebinding.go | 3 + .../serviceaccount/serviceaccount.go | 3 + .../component/snapshotlease/snapshotlease.go | 25 +-- internal/component/statefulset/statefulset.go | 106 +++++++++++- internal/component/types.go | 4 + internal/controller/etcd/reconcile_spec.go | 33 ++++ internal/errors/errors.go | 18 +- internal/utils/conditions.go | 18 ++ internal/utils/conditions_test.go | 137 ++++++++++++++++ internal/utils/labels.go | 18 ++ internal/utils/labels_test.go | 154 ++++++++++++++++++ internal/utils/miscellaneous.go | 8 - internal/webhook/sentinel/handler.go | 2 +- 19 files changed, 522 insertions(+), 25 deletions(-) create mode 100644 internal/utils/conditions.go create mode 100644 internal/utils/conditions_test.go create mode 100644 internal/utils/labels.go create mode 100644 internal/utils/labels_test.go diff --git a/internal/component/clientservice/clientservice.go b/internal/component/clientservice/clientservice.go index 905bd7b9c..ff0d82ab8 100644 --- a/internal/component/clientservice/clientservice.go +++ b/internal/component/clientservice/clientservice.go @@ -61,6 +61,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the client service component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the client service for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd.ObjectMeta) diff --git a/internal/component/configmap/configmap.go b/internal/component/configmap/configmap.go index c3f4e3c52..320e2e132 100644 --- a/internal/component/configmap/configmap.go +++ b/internal/component/configmap/configmap.go @@ -63,6 +63,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the configmap component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the configmap for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { cm := emptyConfigMap(getObjectKey(etcd.ObjectMeta)) diff --git a/internal/component/memberlease/memberlease.go b/internal/component/memberlease/memberlease.go index 8ff4caa89..3fb49275a 100644 --- a/internal/component/memberlease/memberlease.go +++ b/internal/component/memberlease/memberlease.go @@ -63,6 +63,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the member lease component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the member leases for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKeys := getObjectKeys(etcd) diff --git a/internal/component/peerservice/peerservice.go b/internal/component/peerservice/peerservice.go index 6d9f62ef1..54e76f667 100644 --- a/internal/component/peerservice/peerservice.go +++ b/internal/component/peerservice/peerservice.go @@ -61,6 +61,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the peer service component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the peer service for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd.ObjectMeta) diff --git a/internal/component/poddistruptionbudget/poddisruptionbudget.go b/internal/component/poddistruptionbudget/poddisruptionbudget.go index c0c6dd7a8..265827893 100644 --- a/internal/component/poddistruptionbudget/poddisruptionbudget.go +++ b/internal/component/poddistruptionbudget/poddisruptionbudget.go @@ -61,6 +61,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the pod disruption budget component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the pod disruption budget for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd.ObjectMeta) diff --git a/internal/component/role/role.go b/internal/component/role/role.go index ccd231cc1..edc327464 100644 --- a/internal/component/role/role.go +++ b/internal/component/role/role.go @@ -61,6 +61,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the role component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the role for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd.ObjectMeta) diff --git a/internal/component/rolebinding/rolebinding.go b/internal/component/rolebinding/rolebinding.go index 9be850052..d93541e5c 100644 --- a/internal/component/rolebinding/rolebinding.go +++ b/internal/component/rolebinding/rolebinding.go @@ -61,6 +61,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the role binding component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the role binding for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd.ObjectMeta) diff --git a/internal/component/serviceaccount/serviceaccount.go b/internal/component/serviceaccount/serviceaccount.go index dbbf61281..5ed071fa5 100644 --- a/internal/component/serviceaccount/serviceaccount.go +++ b/internal/component/serviceaccount/serviceaccount.go @@ -63,6 +63,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the service account component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the service account for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { objectKey := getObjectKey(etcd.ObjectMeta) diff --git a/internal/component/snapshotlease/snapshotlease.go b/internal/component/snapshotlease/snapshotlease.go index e658bf894..7b74350fd 100644 --- a/internal/component/snapshotlease/snapshotlease.go +++ b/internal/component/snapshotlease/snapshotlease.go @@ -51,12 +51,11 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO deltaSnapshotObjectKey := client.ObjectKey{Name: druidv1alpha1.GetDeltaSnapshotLeaseName(etcdObjMeta), Namespace: etcdObjMeta.Namespace} deltaSnapshotLease, err := r.getLeasePartialObjectMetadata(ctx, deltaSnapshotObjectKey) if err != nil { - return resourceNames, &druiderr.DruidError{ - Code: ErrGetSnapshotLease, - Cause: err, - Operation: "GetExistingResourceNames", - Message: fmt.Sprintf("Error getting delta snapshot lease: %v for etcd: %v", deltaSnapshotObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), - } + return resourceNames, druiderr.WrapError(err, + ErrGetSnapshotLease, + "GetExistingResourceNames", + fmt.Sprintf("Error getting delta snapshot lease: %v for etcd: %v", deltaSnapshotObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), + ) } if deltaSnapshotLease != nil && metav1.IsControlledBy(deltaSnapshotLease, &etcdObjMeta) { resourceNames = append(resourceNames, deltaSnapshotLease.Name) @@ -64,12 +63,11 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO fullSnapshotObjectKey := client.ObjectKey{Name: druidv1alpha1.GetFullSnapshotLeaseName(etcdObjMeta), Namespace: etcdObjMeta.Namespace} fullSnapshotLease, err := r.getLeasePartialObjectMetadata(ctx, fullSnapshotObjectKey) if err != nil { - return resourceNames, &druiderr.DruidError{ - Code: ErrGetSnapshotLease, - Cause: err, - Operation: "GetExistingResourceNames", - Message: fmt.Sprintf("Error getting full snapshot lease: %v for etcd: %v", fullSnapshotObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), - } + return resourceNames, druiderr.WrapError(err, + ErrGetSnapshotLease, + "GetExistingResourceNames", + fmt.Sprintf("Error getting full snapshot lease: %v for etcd: %v", fullSnapshotObjectKey, druidv1alpha1.GetNamespaceName(etcdObjMeta)), + ) } if fullSnapshotLease != nil && metav1.IsControlledBy(fullSnapshotLease, &etcdObjMeta) { resourceNames = append(resourceNames, fullSnapshotLease.Name) @@ -77,6 +75,9 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync is a no-op for the snapshot lease component. +func (r _resource) PreSync(_ component.OperatorContext, _ *druidv1alpha1.Etcd) error { return nil } + // Sync creates or updates the snapshot leases for the given Etcd. // If backups are disabled for the Etcd resource, any existing snapshot leases are deleted. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index 30adf83bc..b960fee17 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -13,11 +13,14 @@ import ( druiderr "github.com/gardener/etcd-druid/internal/errors" "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -25,6 +28,8 @@ import ( const ( // ErrGetStatefulSet indicates an error in getting the statefulset resource. ErrGetStatefulSet druidv1alpha1.ErrorCode = "ERR_GET_STATEFULSET" + // ErrPreSyncStatefulSet indicates an error in pre-sync operations for the statefulset resource. + ErrPreSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_PRESYNC_STATEFULSET" // ErrSyncStatefulSet indicates an error in syncing the statefulset resource. ErrSyncStatefulSet druidv1alpha1.ErrorCode = "ERR_SYNC_STATEFULSET" // ErrDeleteStatefulSet indicates an error in deleting the statefulset resource. @@ -67,6 +72,58 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO return resourceNames, nil } +// PreSync recreates the statefulset for the given Etcd, if label selector for the existing statefulset +// is different from the label selector required to be applied on it. This is because the statefulset's +// spec.selector field is immutable and cannot be updated on the existing statefulset. +func (r _resource) PreSync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { + sts, err := r.getExistingStatefulSet(ctx, etcd.ObjectMeta) + if err != nil { + return druiderr.WrapError(err, + ErrPreSyncStatefulSet, + "PreSync", + fmt.Sprintf("Error getting StatefulSet: %v for etcd: %v", getObjectKey(etcd.ObjectMeta), druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) + } + // if no sts exists, this method is a no-op. + if sts == nil { + return nil + } + + // patch sts with new pod labels. + if err = r.checkAndPatchStsPodLabelsOnMismatch(ctx, etcd, sts); err != nil { + return druiderr.WrapError(err, + ErrPreSyncStatefulSet, + "PreSync", + fmt.Sprintf("Error checking and patching StatefulSet pods with new labels for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) + } + + // check if pods have been updated with new labels and become ready. + podsUpdatedAndReady, err := r.areStatefulSetPodsUpdatedAndReady(ctx, sts) + if err != nil { + return druiderr.WrapError(err, + ErrPreSyncStatefulSet, + "PreSync", + fmt.Sprintf("Error checking if StatefulSet pods are updated for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) + } + if !podsUpdatedAndReady { + errMessage := fmt.Sprintf("StatefulSet pods are not yet updated with new pod labels and ready, for StatefulSet: %v for etcd: %v", getObjectKey(sts.ObjectMeta), druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)) + return druiderr.WrapError(fmt.Errorf(errMessage), + druiderr.ErrRetriable, + "PreSync", + errMessage, + ) + } + + // if sts label selector needs to be changed, then delete the statefulset, but keeping the pods intact. + if err = r.checkAndDeleteStsWithOrphansOnLabelSelectorMismatch(ctx, etcd, sts); err != nil { + return druiderr.WrapError(err, + ErrPreSyncStatefulSet, + "PreSync", + fmt.Sprintf("Error checking and deleting StatefulSet with orphans for etcd: %v", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) + } + + return nil +} + // Sync creates or updates the statefulset for the given Etcd. func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { var ( @@ -180,8 +237,8 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru return nil } -func isStatefulSetPatchedWithPeerTLSVolMount(existingSts *appsv1.StatefulSet) bool { - volumes := existingSts.Spec.Template.Spec.Volumes +func isStatefulSetPatchedWithPeerTLSVolMount(sts *appsv1.StatefulSet) bool { + volumes := sts.Spec.Template.Spec.Volumes var peerURLCAEtcdVolPresent, peerURLEtcdServerTLSVolPresent bool for _, vol := range volumes { if vol.Name == common.VolumeNameEtcdPeerCA { @@ -199,6 +256,51 @@ func isPeerTLSEnablementPending(peerTLSEnabledStatusFromMembers bool, etcd *drui return !peerTLSEnabledStatusFromMembers && etcd.Spec.Etcd.PeerUrlTLS != nil } +func (r _resource) checkAndPatchStsPodLabelsOnMismatch(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) error { + desiredPodLabels := utils.MergeMaps(etcd.Spec.Labels, getStatefulSetLabels(etcd.Name)) + if !utils.ContainsAllDesiredLabels(sts.Spec.Template.Labels, desiredPodLabels) { + ctx.Logger.Info("Patching StatefulSet with new pod labels", "objectKey", getObjectKey(etcd.ObjectMeta)) + originalSts := sts.DeepCopy() + sts.Spec.Template.Labels = utils.MergeMaps(sts.Spec.Template.Labels, desiredPodLabels) + if err := r.client.Patch(ctx, sts, client.MergeFrom(originalSts)); err != nil { + return err + } + } + return nil +} + +func (r _resource) areStatefulSetPodsUpdatedAndReady(ctx component.OperatorContext, sts *appsv1.StatefulSet) (bool, error) { + if sts.Status.UpdatedReplicas != *sts.Spec.Replicas { + return false, nil + } + podList := &corev1.PodList{} + if err := r.client.List(ctx, podList, client.InNamespace(sts.Namespace), client.MatchingLabels(sts.Spec.Template.Labels)); err != nil { + return false, err + } + if len(podList.Items) != int(*sts.Spec.Replicas) { + return false, nil + } + for _, pod := range podList.Items { + if !utils.ContainsLabel(pod.Labels, appsv1.StatefulSetRevisionLabel, sts.Status.UpdateRevision) { + return false, nil + } + if !utils.HasPodReadyConditionTrue(&pod) { + return false, nil + } + } + return true, nil +} + +func (r _resource) checkAndDeleteStsWithOrphansOnLabelSelectorMismatch(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) error { + if !labels.Equals(sts.Spec.Selector.MatchLabels, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)) { + ctx.Logger.Info("Deleting StatefulSet for recreation later, as label selector has changed", "objectKey", getObjectKey(etcd.ObjectMeta)) + if err := r.client.Delete(ctx, sts, client.PropagationPolicy(metav1.DeletePropagationOrphan)); err != nil { + return err + } + } + return nil +} + func emptyStatefulSet(obj metav1.ObjectMeta) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/component/types.go b/internal/component/types.go index 2ed478ec8..ea62f5ee2 100644 --- a/internal/component/types.go +++ b/internal/component/types.go @@ -48,4 +48,8 @@ type Operator interface { // create it. If there are changes in the owning Etcd resource that transpires changes to one or more resources // managed by this Operator then those component(s) will be either be updated or a deletion is triggered. Sync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error + // PreSync performs any preparatory operations that are required before the actual sync operation is performed, + // to bring the component to a sync-ready state. If the sync-ready state is already satisfied, + // then this method will be a no-op. + PreSync(ctx OperatorContext, etcd *druidv1alpha1.Etcd) error } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index ee3ae1b5d..4100fb48a 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -5,10 +5,14 @@ package etcd import ( + "fmt" + "time" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/component" ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils" + druiderr "github.com/gardener/etcd-druid/internal/errors" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" @@ -18,10 +22,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) +const retryInterval = 1 * time.Second + func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { reconcileStepFns := []reconcileFn{ r.recordReconcileStartOperation, r.ensureFinalizer, + r.preSyncEtcdResources, r.syncEtcdResources, r.updateObservedGeneration, r.recordReconcileSuccessOperation, @@ -75,6 +82,26 @@ func (r *Reconciler) ensureFinalizer(ctx component.OperatorContext, etcdObjKey c return ctrlutils.ContinueReconcile() } +func (r *Reconciler) preSyncEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { + etcd := &druidv1alpha1.Etcd{} + if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { + return result + } + resourceOperators := r.getOrderedOperatorsForPreSync() + for _, kind := range resourceOperators { + op := r.operatorRegistry.GetOperator(kind) + if err := op.PreSync(ctx, etcd); err != nil { + if druiderr.IsRetriable(err) { + ctx.Logger.Info("retrying pre-sync of component", "kind", kind, "retryInterval", retryInterval.String()) + return ctrlutils.ReconcileAfter(retryInterval, fmt.Sprintf("retrying pre-sync of component %s after %s", kind, retryInterval.String())) + } + ctx.Logger.Error(err, "failed to sync etcd resource", "kind", kind) + return ctrlutils.ReconcileWithError(err) + } + } + return ctrlutils.ContinueReconcile() +} + func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult { etcd := &druidv1alpha1.Etcd{} if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) { @@ -181,6 +208,12 @@ func (r *Reconciler) recordEtcdSpecReconcileSuspension(etcd *druidv1alpha1.Etcd, ) } +func (r *Reconciler) getOrderedOperatorsForPreSync() []component.Kind { + return []component.Kind{ + component.StatefulSetKind, + } +} + func (r *Reconciler) getOrderedOperatorsForSync() []component.Kind { return []component.Kind{ component.MemberLeaseKind, diff --git a/internal/errors/errors.go b/internal/errors/errors.go index e519315b6..1bca8b16c 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -13,6 +13,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// ErrRetriable is a special error code that indicates that the current step should be retried, +// as some conditions during reconciliation are not met. This should not be used in case +// there is an actual error during reconciliation. +const ErrRetriable = "ERR_RETRIABLE" + // DruidError is a custom error that should be used throughout druid which encapsulates // the underline error (cause) along with error code and contextual information captured // as operation during which an error occurred and any custom message. @@ -29,8 +34,17 @@ type DruidError struct { ObservedAt time.Time } -func (r *DruidError) Error() string { - return fmt.Sprintf("[Operation: %s, Code: %s] %s", r.Operation, r.Code, r.Cause.Error()) +func (e *DruidError) Error() string { + return fmt.Sprintf("[Operation: %s, Code: %s] %s", e.Operation, e.Code, e.Cause.Error()) +} + +// IsRetriable checks if the given error is of type DruidError and has the given error code. +func IsRetriable(err error) bool { + druidErr := &DruidError{} + if errors.As(err, &druidErr) { + return druidErr.Code == ErrRetriable + } + return false } // WrapError wraps an error and contextual information like code, operation and message to create a DruidError diff --git a/internal/utils/conditions.go b/internal/utils/conditions.go new file mode 100644 index 000000000..00cb617b3 --- /dev/null +++ b/internal/utils/conditions.go @@ -0,0 +1,18 @@ +package utils + +import corev1 "k8s.io/api/core/v1" + +// MatchPodConditions checks if the specified condition type and status are present in the given pod conditions. +func MatchPodConditions(conditions []corev1.PodCondition, condType corev1.PodConditionType, condStatus corev1.ConditionStatus) bool { + for _, condition := range conditions { + if condition.Type == condType && condition.Status == condStatus { + return true + } + } + return false +} + +// HasPodReadyConditionTrue checks if the pod has a Ready condition with status True. +func HasPodReadyConditionTrue(pod *corev1.Pod) bool { + return MatchPodConditions(pod.Status.Conditions, corev1.PodReady, corev1.ConditionTrue) +} diff --git a/internal/utils/conditions_test.go b/internal/utils/conditions_test.go new file mode 100644 index 000000000..38fe3fcbd --- /dev/null +++ b/internal/utils/conditions_test.go @@ -0,0 +1,137 @@ +package utils + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + + . "github.com/onsi/gomega" +) + +func TestMatchPodConditions(t *testing.T) { + testCases := []struct { + name string + conditions []corev1.PodCondition + condType corev1.PodConditionType + condStatus corev1.ConditionStatus + expected bool + }{ + { + name: "condition type and status are present", + conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + condType: corev1.PodReady, + condStatus: corev1.ConditionTrue, + expected: true, + }, + { + name: "condition type and status are not present", + conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }, + }, + condType: corev1.PodReady, + condStatus: corev1.ConditionTrue, + expected: false, + }, + { + name: "condition type is present but status is not", + conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionUnknown, + }, + }, + condType: corev1.PodReady, + condStatus: corev1.ConditionTrue, + expected: false, + }, + { + name: "condition type is not present but status is", + conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + }, + }, + condType: corev1.PodReady, + condStatus: corev1.ConditionTrue, + expected: false, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g.Expect(MatchPodConditions(tc.conditions, tc.condType, tc.condStatus)).To(Equal(tc.expected)) + }) + } +} + +func TestHasPodReadyConditionTrue(t *testing.T) { + testCases := []struct { + name string + pod *corev1.Pod + expected bool + }{ + { + name: "pod has Ready condition with status True", + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + expected: true, + }, + { + name: "pod has Ready condition with status not True", + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }, + }, + }, + }, + expected: false, + }, + { + name: "pod has no Ready condition", + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + expected: false, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g.Expect(HasPodReadyConditionTrue(tc.pod)).To(Equal(tc.expected)) + }) + } +} diff --git a/internal/utils/labels.go b/internal/utils/labels.go new file mode 100644 index 000000000..270a9db60 --- /dev/null +++ b/internal/utils/labels.go @@ -0,0 +1,18 @@ +package utils + +// ContainsAllDesiredLabels checks if the actual map contains all the desired labels. +func ContainsAllDesiredLabels(actual, desired map[string]string) bool { + for key, desiredValue := range desired { + actualValue, ok := actual[key] + if !ok || actualValue != desiredValue { + return false + } + } + return true +} + +// ContainsLabel checks if the actual map contains the specified key-value pair. +func ContainsLabel(actual map[string]string, key, value string) bool { + actualValue, ok := actual[key] + return ok && actualValue == value +} diff --git a/internal/utils/labels_test.go b/internal/utils/labels_test.go new file mode 100644 index 000000000..bbe69d84f --- /dev/null +++ b/internal/utils/labels_test.go @@ -0,0 +1,154 @@ +package utils + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestContainsAllDesiredLabels(t *testing.T) { + testCases := []struct { + name string + actual map[string]string + desired map[string]string + expected bool + }{ + { + name: "actual map is missing a key", + actual: map[string]string{ + "key1": "value1", + }, + desired: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expected: false, + }, + { + name: "actual map has a key with a different value", + actual: map[string]string{ + "key1": "value1", + "key2": "value3", + }, + desired: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expected: false, + }, + { + name: "actual map has all desired labels", + actual: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + desired: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expected: true, + }, + { + name: "actual map has all desired labels and extra labels", + actual: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value3", + }, + desired: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expected: true, + }, + { + name: "actual map is nil", + actual: nil, + desired: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expected: false, + }, + { + name: "desired map is nil", + actual: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + desired: nil, + expected: true, + }, + { + name: "both maps are nil", + actual: nil, + desired: nil, + expected: true, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g.Expect(ContainsAllDesiredLabels(tc.actual, tc.desired)).To(Equal(tc.expected)) + }) + } +} + +func TestContainsLabel(t *testing.T) { + testCases := []struct { + name string + labels map[string]string + key string + value string + expected bool + }{ + { + name: "labels map is missing the key", + labels: map[string]string{ + "key1": "value1", + }, + key: "key2", + value: "value2", + expected: false, + }, + { + name: "labels map has the key with a different value", + labels: map[string]string{ + "key1": "value1", + "key2": "value3", + }, + key: "key2", + value: "value2", + expected: false, + }, + { + name: "labels map has the key with the desired value", + labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + key: "key2", + value: "value2", + expected: true, + }, + { + name: "labels map is nil", + labels: nil, + key: "key1", + value: "value1", + expected: false, + }, + } + + g := NewWithT(t) + t.Parallel() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g.Expect(ContainsLabel(tc.labels, tc.key, tc.value)).To(Equal(tc.expected)) + }) + } +} diff --git a/internal/utils/miscellaneous.go b/internal/utils/miscellaneous.go index d6856d15d..b116019de 100644 --- a/internal/utils/miscellaneous.go +++ b/internal/utils/miscellaneous.go @@ -12,14 +12,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// MergeStringMaps merges the content of the newMaps with the oldMap. If a key already exists then -// it gets overwritten by the last value with the same key. -func MergeStringMaps(oldMap map[string]string, newMaps ...map[string]string) map[string]string { - allMaps := []map[string]string{oldMap} - allMaps = append(allMaps, newMaps...) - return MergeMaps(allMaps...) -} - // MergeMaps merges the contents of maps. All maps will be processed in the order // in which they are sent. For overlapping keys across source maps, value in the merged map // for this key will be from the last occurrence of the key-value. diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 4a104a2f0..650c9eaa0 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -56,7 +56,7 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { requestGKString := fmt.Sprintf("%s/%s", req.Kind.Group, req.Kind.Kind) log := h.logger.WithValues("name", req.Name, "namespace", req.Namespace, "resourceGroupKind", requestGKString, "operation", req.Operation, "user", req.UserInfo.Username) - log.Info("Sentinel webhook invoked") + log.V(1).Info("Sentinel webhook invoked") if slices.Contains(allowedOperations, req.Operation) { return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) From 0d96252ffa7e6e43ab011515d48d52c4ae82a2c0 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 31 May 2024 18:20:37 +0530 Subject: [PATCH 216/235] Add tests for internal/errors package, add license headers --- internal/errors/errors.go | 5 ++- internal/errors/errors_test.go | 67 +++++++++++++++++++++++++++++++ internal/utils/conditions.go | 4 ++ internal/utils/conditions_test.go | 4 ++ internal/utils/labels.go | 4 ++ internal/utils/labels_test.go | 4 ++ 6 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 internal/errors/errors_test.go diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 1bca8b16c..6841b3c40 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -69,7 +69,10 @@ func MapToLastErrors(errs []error) []druidv1alpha1.LastError { for _, err := range errs { druidErr := &DruidError{} if errors.As(err, &druidErr) { - desc := fmt.Sprintf("[Operation: %s, Code: %s] message: %s, cause: %s", druidErr.Operation, druidErr.Code, druidErr.Message, druidErr.Cause.Error()) + desc := fmt.Sprintf("[Operation: %s, Code: %s] message: %s", druidErr.Operation, druidErr.Code, druidErr.Message) + if druidErr.Cause != nil { + desc += fmt.Sprintf(", cause: %s", druidErr.Cause.Error()) + } lastErr := druidv1alpha1.LastError{ Code: druidErr.Code, Description: desc, diff --git a/internal/errors/errors_test.go b/internal/errors/errors_test.go new file mode 100644 index 000000000..5d0aaa19c --- /dev/null +++ b/internal/errors/errors_test.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package errors + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/gomega" +) + +func TestWrapError(t *testing.T) { + err := WrapError(fmt.Errorf("testError"), "ERR_TEST", "testOp", "testMsg") + + g := NewWithT(t) + druidErr, ok := err.(*DruidError) + g.Expect(ok).To(BeTrue()) + g.Expect(string(druidErr.Code)).To(Equal("ERR_TEST")) + g.Expect(druidErr.Operation).To(Equal("testOp")) + g.Expect(druidErr.Message).To(Equal("testMsg")) +} + +func TestIsRetriable(t *testing.T) { + err := &DruidError{ + Code: ErrRetriable, + Cause: nil, + Operation: "", + Message: "", + ObservedAt: time.Now().UTC(), + } + + g := NewWithT(t) + g.Expect(IsRetriable(err)).To(BeTrue()) + + err.Code = "ERR_TEST" + g.Expect(IsRetriable(err)).To(BeFalse()) +} + +func TestMapToLastErrors(t *testing.T) { + errs := []error{ + &DruidError{ + Code: "ERR_TEST1", + Cause: fmt.Errorf("testError1"), + Operation: "testOp", + Message: "testMsg", + ObservedAt: time.Now().UTC(), + }, + &DruidError{ + Code: "ERR_TEST2", + Cause: nil, + Operation: "testOp", + Message: "testMsg", + ObservedAt: time.Now().UTC(), + }, + } + lastErrs := MapToLastErrors(errs) + + g := NewWithT(t) + g.Expect(len(lastErrs)).To(Equal(2)) + g.Expect(string(lastErrs[0].Code)).To(Equal("ERR_TEST1")) + g.Expect(lastErrs[0].Description).To(Equal("[Operation: testOp, Code: ERR_TEST1] message: testMsg, cause: testError1")) + g.Expect(string(lastErrs[1].Code)).To(Equal("ERR_TEST2")) + g.Expect(lastErrs[1].Description).To(Equal("[Operation: testOp, Code: ERR_TEST2] message: testMsg")) +} diff --git a/internal/utils/conditions.go b/internal/utils/conditions.go index 00cb617b3..147b60cdd 100644 --- a/internal/utils/conditions.go +++ b/internal/utils/conditions.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import corev1 "k8s.io/api/core/v1" diff --git a/internal/utils/conditions_test.go b/internal/utils/conditions_test.go index 38fe3fcbd..5f3aab4be 100644 --- a/internal/utils/conditions_test.go +++ b/internal/utils/conditions_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( diff --git a/internal/utils/labels.go b/internal/utils/labels.go index 270a9db60..697875822 100644 --- a/internal/utils/labels.go +++ b/internal/utils/labels.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils // ContainsAllDesiredLabels checks if the actual map contains all the desired labels. diff --git a/internal/utils/labels_test.go b/internal/utils/labels_test.go index bbe69d84f..f01793798 100644 --- a/internal/utils/labels_test.go +++ b/internal/utils/labels_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package utils import ( From 6cb5488d7cc4d4e6dee1f8a2b8acd74022d49965 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 3 Jun 2024 10:32:00 +0530 Subject: [PATCH 217/235] fixed IT tests after introduction of presync --- api/v1alpha1/helper.go | 5 + internal/component/statefulset/statefulset.go | 24 +- internal/controller/etcd/reconcile_spec.go | 6 +- internal/errors/errors.go | 10 +- internal/errors/errors_test.go | 47 ++-- test/it/controller/etcd/helper.go | 255 ++++++++++++++++++ test/it/controller/etcd/reconciler_test.go | 167 +----------- 7 files changed, 320 insertions(+), 194 deletions(-) create mode 100644 test/it/controller/etcd/helper.go diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index a290aab45..6d95826d9 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -78,6 +78,11 @@ func GetFullSnapshotLeaseName(etcdObjMeta metav1.ObjectMeta) string { return fmt.Sprintf("%s-full-snap", etcdObjMeta.Name) } +// GetStatefulSetName returns the name of the StatefulSet for the Etcd. +func GetStatefulSetName(etcdObjMeta metav1.ObjectMeta) string { + return etcdObjMeta.Name +} + // --------------- Miscellaneous helper functions --------------- // GetNamespaceName is a convenience function which creates a types.NamespacedName for an Etcd resource. diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index b960fee17..48eeb9223 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -107,7 +107,7 @@ func (r _resource) PreSync(ctx component.OperatorContext, etcd *druidv1alpha1.Et if !podsUpdatedAndReady { errMessage := fmt.Sprintf("StatefulSet pods are not yet updated with new pod labels and ready, for StatefulSet: %v for etcd: %v", getObjectKey(sts.ObjectMeta), druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)) return druiderr.WrapError(fmt.Errorf(errMessage), - druiderr.ErrRetriable, + druiderr.ErrRequeueAfter, "PreSync", errMessage, ) @@ -145,10 +145,7 @@ func (r _resource) Sync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) // StatefulSet exists, check if TLS has been enabled for peer communication, if yes then it is currently a multistep // process to ensure that all members are updated and establish peer TLS communication. if err = r.handlePeerTLSChanges(ctx, etcd, existingSTS); err != nil { - return druiderr.WrapError(err, - ErrSyncStatefulSet, - "Sync", - fmt.Sprintf("Error while handling peer URL TLS change for StatefulSet: %v, etcd: %v", objectKey, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) + return err } return r.createOrPatch(ctx, etcd) } @@ -218,7 +215,10 @@ func (r _resource) createOrPatch(ctx component.OperatorContext, etcd *druidv1alp func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, existingSts *appsv1.StatefulSet) error { peerTLSEnabledForMembers, err := utils.IsPeerURLTLSEnabledForMembers(ctx, r.client, ctx.Logger, etcd.Namespace, etcd.Name, *existingSts.Spec.Replicas) if err != nil { - return fmt.Errorf("error checking if peer URL TLS is enabled: %w", err) + return druiderr.WrapError(err, + ErrSyncStatefulSet, + "Sync", + fmt.Sprintf("Error checking if peer TLS is enabled for statefulset: %v, etcd: %v", client.ObjectKeyFromObject(existingSts), client.ObjectKeyFromObject(etcd))) } if isPeerTLSEnablementPending(peerTLSEnabledForMembers, etcd) { @@ -226,12 +226,18 @@ func (r _resource) handlePeerTLSChanges(ctx component.OperatorContext, etcd *dru // This step ensures that only STS is updated with secret volume mounts which gets added to the etcd component due to // enabling of TLS for peer communication. It preserves the current STS replicas. if err = r.createOrPatchWithReplicas(ctx, etcd, *existingSts.Spec.Replicas); err != nil { - return fmt.Errorf("error creating or patching StatefulSet with TLS enabled: %w", err) + return druiderr.WrapError(err, + ErrSyncStatefulSet, + "Sync", + fmt.Sprintf("Error creating or patching StatefulSet with TLS enabled for StatefulSet: %v, etcd: %v", client.ObjectKeyFromObject(existingSts), client.ObjectKeyFromObject(etcd))) } } else { ctx.Logger.Info("Secret volume mounts to enable Peer URL TLS have already been mounted. Skipping patching StatefulSet with secret volume mounts.") } - return fmt.Errorf("peer URL TLS not enabled for #%d members for etcd: %v, requeuing reconcile request", *existingSts.Spec.Replicas, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)) + return druiderr.WrapError(err, + druiderr.ErrRequeueAfter, + "Sync", + fmt.Sprintf("Peer URL TLS not enabled for #%d members for etcd: %v, requeuing reconcile request", existingSts.Spec.Replicas, client.ObjectKeyFromObject(etcd))) } ctx.Logger.Info("Peer URL TLS has been enabled for all currently running members") return nil @@ -304,7 +310,7 @@ func (r _resource) checkAndDeleteStsWithOrphansOnLabelSelectorMismatch(ctx compo func emptyStatefulSet(obj metav1.ObjectMeta) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: obj.Name, + Name: druidv1alpha1.GetStatefulSetName(obj), Namespace: obj.Namespace, OwnerReferences: []metav1.OwnerReference{druidv1alpha1.GetAsOwnerReference(obj)}, }, diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index 4100fb48a..e828a4f4d 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -91,7 +91,7 @@ func (r *Reconciler) preSyncEtcdResources(ctx component.OperatorContext, etcdObj for _, kind := range resourceOperators { op := r.operatorRegistry.GetOperator(kind) if err := op.PreSync(ctx, etcd); err != nil { - if druiderr.IsRetriable(err) { + if druiderr.IsRequeueAfterError(err) { ctx.Logger.Info("retrying pre-sync of component", "kind", kind, "retryInterval", retryInterval.String()) return ctrlutils.ReconcileAfter(retryInterval, fmt.Sprintf("retrying pre-sync of component %s after %s", kind, retryInterval.String())) } @@ -111,6 +111,10 @@ func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey for _, kind := range resourceOperators { op := r.operatorRegistry.GetOperator(kind) if err := op.Sync(ctx, etcd); err != nil { + if druiderr.IsRequeueAfterError(err) { + ctx.Logger.Info("retrying sync of component", "kind", kind, "retryInterval", retryInterval.String()) + return ctrlutils.ReconcileAfter(retryInterval, fmt.Sprintf("retrying sync of component %s after %s", kind, retryInterval.String())) + } ctx.Logger.Error(err, "failed to sync etcd resource", "kind", kind) return ctrlutils.ReconcileWithError(err) } diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 6841b3c40..9c62f78a0 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -13,10 +13,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// ErrRetriable is a special error code that indicates that the current step should be retried, +// ErrRequeueAfter is a special error code that indicates that the current step should be re-queued after a certain time, // as some conditions during reconciliation are not met. This should not be used in case // there is an actual error during reconciliation. -const ErrRetriable = "ERR_RETRIABLE" +const ErrRequeueAfter = "ERR_REQUEUE_AFTER" // DruidError is a custom error that should be used throughout druid which encapsulates // the underline error (cause) along with error code and contextual information captured @@ -38,11 +38,11 @@ func (e *DruidError) Error() string { return fmt.Sprintf("[Operation: %s, Code: %s] %s", e.Operation, e.Code, e.Cause.Error()) } -// IsRetriable checks if the given error is of type DruidError and has the given error code. -func IsRetriable(err error) bool { +// IsRequeueAfterError checks if the given error is of type DruidError and has the given error code. +func IsRequeueAfterError(err error) bool { druidErr := &DruidError{} if errors.As(err, &druidErr) { - return druidErr.Code == ErrRetriable + return druidErr.Code == ErrRequeueAfter } return false } diff --git a/internal/errors/errors_test.go b/internal/errors/errors_test.go index 5d0aaa19c..39f2121d4 100644 --- a/internal/errors/errors_test.go +++ b/internal/errors/errors_test.go @@ -5,51 +5,68 @@ package errors import ( + "errors" "fmt" "testing" "time" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" . "github.com/onsi/gomega" ) func TestWrapError(t *testing.T) { err := WrapError(fmt.Errorf("testError"), "ERR_TEST", "testOp", "testMsg") - g := NewWithT(t) - druidErr, ok := err.(*DruidError) - g.Expect(ok).To(BeTrue()) + druidErr := &DruidError{} + g.Expect(errors.As(err, &druidErr)).To(BeTrue()) g.Expect(string(druidErr.Code)).To(Equal("ERR_TEST")) g.Expect(druidErr.Operation).To(Equal("testOp")) g.Expect(druidErr.Message).To(Equal("testMsg")) } -func TestIsRetriable(t *testing.T) { - err := &DruidError{ - Code: ErrRetriable, - Cause: nil, - Operation: "", - Message: "", - ObservedAt: time.Now().UTC(), +func TestIsRequeueAfterError(t *testing.T) { + testCases := []struct { + name string + code druidv1alpha1.ErrorCode + expected bool + }{ + { + name: "error has code ERR_REQUEUE_AFTER", + code: ErrRequeueAfter, + expected: true, + }, + { + name: "error has code ERR_TEST", + code: druidv1alpha1.ErrorCode("ERR_TEST"), + expected: false, + }, } g := NewWithT(t) - g.Expect(IsRetriable(err)).To(BeTrue()) + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := &DruidError{ + Code: tc.code, + ObservedAt: time.Now().UTC(), + } + g.Expect(IsRequeueAfterError(err)).To(Equal(tc.expected)) - err.Code = "ERR_TEST" - g.Expect(IsRetriable(err)).To(BeFalse()) + }) + } } func TestMapToLastErrors(t *testing.T) { errs := []error{ &DruidError{ - Code: "ERR_TEST1", + Code: druidv1alpha1.ErrorCode("ERR_TEST1"), Cause: fmt.Errorf("testError1"), Operation: "testOp", Message: "testMsg", ObservedAt: time.Now().UTC(), }, &DruidError{ - Code: "ERR_TEST2", + Code: druidv1alpha1.ErrorCode("ERR_TEST2"), Cause: nil, Operation: "testOp", Message: "testMsg", diff --git a/test/it/controller/etcd/helper.go b/test/it/controller/etcd/helper.go new file mode 100644 index 000000000..ec0321514 --- /dev/null +++ b/test/it/controller/etcd/helper.go @@ -0,0 +1,255 @@ +package etcd + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "strconv" + "testing" + "time" + + "github.com/gardener/etcd-druid/internal/utils" + "github.com/gardener/etcd-druid/test/it/controller/assets" + "github.com/gardener/etcd-druid/test/it/setup" + . "github.com/onsi/gomega" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/common" + "github.com/gardener/etcd-druid/internal/controller/etcd" + "github.com/gardener/etcd-druid/internal/features" + testutils "github.com/gardener/etcd-druid/test/utils" + "github.com/gardener/gardener/pkg/controllerutils" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/featuregate" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const testNamespacePrefix = "etcd-reconciler-test-" + +func createTestNamespaceName(t *testing.T) string { + namespaceSuffix := generateRandomAlphanumericString(t, 4) + return fmt.Sprintf("%s-%s", testNamespacePrefix, namespaceSuffix) +} + +func generateRandomAlphanumericString(t *testing.T, length int) string { + b := make([]byte, length) + _, err := rand.Read(b) + g := NewWithT(t) + g.Expect(err).ToNot(HaveOccurred()) + return hex.EncodeToString(b) +} + +func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTestEnv, autoReconcile bool, clientBuilder *testutils.TestClientBuilder) ReconcilerTestEnv { + g := NewWithT(t) + var ( + reconciler *etcd.Reconciler + err error + ) + g.Expect(itTestEnv.CreateManager(clientBuilder)).To(Succeed()) + itTestEnv.RegisterReconciler(func(mgr manager.Manager) { + reconciler, err = etcd.NewReconcilerWithImageVector(mgr, + &etcd.Config{ + Workers: 5, + EnableEtcdSpecAutoReconcile: autoReconcile, + DisableEtcdServiceAccountAutomount: false, + EtcdStatusSyncPeriod: 2 * time.Second, + FeatureGates: map[featuregate.Feature]bool{ + features.UseEtcdWrapper: true, + }, + EtcdMember: etcd.MemberConfig{ + NotReadyThreshold: 5 * time.Minute, + UnknownThreshold: 1 * time.Minute, + }, + }, assets.CreateImageVector(g)) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(reconciler.RegisterWithManager(mgr)).To(Succeed()) + }) + g.Expect(itTestEnv.StartManager()).To(Succeed()) + t.Log("successfully registered etcd reconciler with manager and started manager") + return ReconcilerTestEnv{ + itTestEnv: itTestEnv, + reconciler: reconciler, + } +} + +func createAndAssertEtcdAndAllManagedResources(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { + g := NewWithT(t) + cl := reconcilerTestEnv.itTestEnv.GetClient() + // create etcdInstance resource + g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) + t.Logf("trigggered creation of etcd instance: {name: %s, namespace: %s}, waiting for resources to be created...", etcdInstance.Name, etcdInstance.Namespace) + // ascertain that all etcd resources are created + assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 3*time.Minute, 2*time.Second) + t.Logf("successfully created all resources for etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) + + // In envtest KCM does not run. As a consequence StatefulSet controller also does not run which results in no pods being created. + // Status of StatefulSet also does not get updated. During reconciliation in StatefulSet component preSync method both StatefulSet + // and Pod status are checked. In order for the IT tests to function correctly both the StatefulSet status needs to be updated + // and pods needs to be explicitly created. + stsUpdateRevision := updateAndGetStsRevision(ctx, t, cl, etcdInstance) + t.Logf("successfully updated sts revision for etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) + createStsPods(ctx, t, cl, etcdInstance.ObjectMeta, stsUpdateRevision) + t.Logf("successfully created pods for statefulset of etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) + + //// add finalizer + //addFinalizer(ctx, g, cl, client.ObjectKeyFromObject(etcdInstance)) + //t.Logf("successfully added finalizer to etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) +} + +func updateAndGetStsRevision(ctx context.Context, t *testing.T, cl client.Client, etcdInstance *druidv1alpha1.Etcd) string { + g := NewWithT(t) + sts := &appsv1.StatefulSet{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: druidv1alpha1.GetStatefulSetName(etcdInstance.ObjectMeta), Namespace: etcdInstance.Namespace}, sts)).To(Succeed()) + originalSts := sts.DeepCopy() + sts.Status.Replicas = etcdInstance.Spec.Replicas + sts.Status.ReadyReplicas = etcdInstance.Spec.Replicas + sts.Status.UpdatedReplicas = etcdInstance.Spec.Replicas + sts.Status.UpdateRevision = fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 4)) + g.Expect(cl.Status().Patch(ctx, sts, client.MergeFrom(originalSts))).To(Succeed()) + return sts.Status.UpdateRevision +} + +func createStsPods(ctx context.Context, t *testing.T, cl client.Client, etcdObjectMeta metav1.ObjectMeta, stsUpdateRevision string) { + g := NewWithT(t) + sts := &appsv1.StatefulSet{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: druidv1alpha1.GetStatefulSetName(etcdObjectMeta), Namespace: etcdObjectMeta.Namespace}, sts)).To(Succeed()) + stsReplicas := *sts.Spec.Replicas + for i := 0; i < int(stsReplicas); i++ { + podName := fmt.Sprintf("%s-%d", sts.Name, i) + podLabels := utils.MergeMaps(sts.Spec.Template.Labels, etcdObjectMeta.Labels, map[string]string{ + "apps.kubernetes.io/pod-index": strconv.Itoa(i), + "statefulset.kubernetes.io/pod-name": podName, + "controller-revision-hash": stsUpdateRevision, + }) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: etcdObjectMeta.Namespace, + Labels: podLabels, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: sts.APIVersion, + Kind: sts.Kind, + BlockOwnerDeletion: pointer.Bool(true), + Name: sts.Name, + UID: sts.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "etcd", + Image: "etcd-wrapper:latest", + }, + }, + }, + } + g.Expect(cl.Create(ctx, pod)).To(Succeed()) + t.Logf("successfully created pod: %s", pod.Name) + // update pod status and set ready condition to true + pod.Status.Conditions = []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + } + g.Expect(cl.Status().Update(ctx, pod)).To(Succeed()) + t.Logf("successfully updated status of pod: %s with ready-condition set to true", pod.Name) + } +} + +func createAndAssertEtcdReconciliation(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { + // create etcd and assert etcd spec reconciliation + createAndAssertEtcdAndAllManagedResources(ctx, t, reconcilerTestEnv, etcdInstance) + + // assert etcd status reconciliation + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) + expectedLastOperation := &druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: druidv1alpha1.LastOperationStateSucceeded, + } + assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, nil, 5*time.Second, 1*time.Second) + assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), false, 5*time.Second, 1*time.Second) + t.Logf("successfully reconciled status of etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) +} + +func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey client.ObjectKey) { + etcdInstance := &druidv1alpha1.Etcd{} + g.Expect(cl.Get(ctx, etcdObjectKey, etcdInstance)).To(Succeed()) + g.Expect(controllerutils.AddFinalizers(ctx, cl, etcdInstance, common.FinalizerName)).To(Succeed()) +} + +type etcdMemberLeaseConfig struct { + name string + memberID string + role druidv1alpha1.EtcdRole + renewTime *metav1.MicroTime +} + +func updateMemberLeaseSpec(ctx context.Context, t *testing.T, cl client.Client, namespace string, memberLeaseConfigs []etcdMemberLeaseConfig) { + g := NewWithT(t) + for _, config := range memberLeaseConfigs { + lease := &coordinationv1.Lease{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: config.name, Namespace: namespace}, lease)).To(Succeed()) + updatedLease := lease.DeepCopy() + updatedLease.Spec.HolderIdentity = pointer.String(fmt.Sprintf("%s:%s", config.memberID, config.role)) + updatedLease.Spec.RenewTime = config.renewTime + g.Expect(cl.Update(ctx, updatedLease)).To(Succeed()) + t.Logf("successfully updated member lease %s with holderIdentity: %s", config.name, *updatedLease.Spec.HolderIdentity) + } +} + +func createPVCs(ctx context.Context, t *testing.T, cl client.Client, sts *appsv1.StatefulSet) []*corev1.PersistentVolumeClaim { + g := NewWithT(t) + pvcs := make([]*corev1.PersistentVolumeClaim, 0, int(*sts.Spec.Replicas)) + volClaimName := sts.Spec.VolumeClaimTemplates[0].Name + for i := 0; i < int(*sts.Spec.Replicas); i++ { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%d", volClaimName, sts.Name, i), + Namespace: sts.Namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + g.Expect(cl.Create(ctx, pvc)).To(Succeed()) + pvcs = append(pvcs, pvc) + t.Logf("successfully created pvc: %s", pvc.Name) + } + return pvcs +} + +func createPVCWarningEvent(ctx context.Context, t *testing.T, cl client.Client, namespace, pvcName, reason, message string) { + g := NewWithT(t) + event := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-event-", pvcName), + Namespace: namespace, + }, + InvolvedObject: corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: pvcName, + Namespace: namespace, + APIVersion: "v1", + }, + Reason: reason, + Message: message, + Type: corev1.EventTypeWarning, + } + g.Expect(cl.Create(ctx, event)).To(Succeed()) + t.Logf("successfully created warning event for pvc: %s", pvcName) +} diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 3901ec5e3..230618d98 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -6,8 +6,6 @@ package etcd import ( "context" - "crypto/rand" - "encoding/hex" "fmt" "os" "testing" @@ -16,29 +14,19 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/component" - "github.com/gardener/etcd-druid/internal/controller/etcd" - "github.com/gardener/etcd-druid/internal/features" "github.com/gardener/etcd-druid/test/it/controller/assets" "github.com/gardener/etcd-druid/test/it/setup" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/controllerutils" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/component-base/featuregate" testclock "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" ) -const testNamespacePrefix = "etcd-reconciler-test-" - var ( sharedITTestEnv setup.IntegrationTestEnv ) @@ -72,7 +60,7 @@ func TestEtcdReconcileSpecWithNoAutoReconcile(t *testing.T) { {"should succeed only in creation of some resources and not all and should record error in lastErrors and lastOperation", testFailureToCreateAllResources}, {"should not reconcile spec when reconciliation is suspended", testWhenReconciliationIsSuspended}, {"should not reconcile upon etcd spec updation when no reconcile operation annotation is set", testEtcdSpecUpdateWhenNoReconcileOperationAnnotationIsSet}, - {"should reconcile upon etcd spec updation when reconcile operation annotation is set", testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet}, + {"should reconcile upon etcd spec update when reconcile operation annotation is set", testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet}, } g := NewWithT(t) for _, test := range tests { @@ -257,8 +245,8 @@ func testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet(t *testing.T, testN g.Expect(cl.Patch(ctx, etcdInstance, client.MergeFrom(originalEtcdInstance))).To(Succeed()) // ***************** test etcd spec reconciliation ***************** - assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 30*time.Second, 2*time.Second) - assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(2), 30*time.Second, 1*time.Second) + assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 30*time.Minute, 2*time.Second) + assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(2), 2*time.Minute, 1*time.Second) expectedLastOperation := &druidv1alpha1.LastOperation{ Type: druidv1alpha1.LastOperationTypeReconcile, State: druidv1alpha1.LastOperationStateSucceeded, @@ -535,152 +523,3 @@ func testEtcdStatusIsInSyncWithStatefulSetStatusWhenCurrentRevisionIsOlderThanUp // assert etcd status assertETCDStatusFieldsDerivedFromStatefulSet(ctx, t, cl, client.ObjectKeyFromObject(etcd), client.ObjectKeyFromObject(stsCopy), 2*time.Minute, 2*time.Second, false) } - -// ------------------------- Helper functions ------------------------- -func createTestNamespaceName(t *testing.T) string { - namespaceSuffix := generateRandomAlphanumericString(t, 4) - return fmt.Sprintf("%s-%s", testNamespacePrefix, namespaceSuffix) -} - -func generateRandomAlphanumericString(t *testing.T, length int) string { - b := make([]byte, length) - _, err := rand.Read(b) - g := NewWithT(t) - g.Expect(err).ToNot(HaveOccurred()) - return hex.EncodeToString(b) -} - -func initializeEtcdReconcilerTestEnv(t *testing.T, itTestEnv setup.IntegrationTestEnv, autoReconcile bool, clientBuilder *testutils.TestClientBuilder) ReconcilerTestEnv { - g := NewWithT(t) - var ( - reconciler *etcd.Reconciler - err error - ) - g.Expect(itTestEnv.CreateManager(clientBuilder)).To(Succeed()) - itTestEnv.RegisterReconciler(func(mgr manager.Manager) { - reconciler, err = etcd.NewReconcilerWithImageVector(mgr, - &etcd.Config{ - Workers: 5, - EnableEtcdSpecAutoReconcile: autoReconcile, - DisableEtcdServiceAccountAutomount: false, - EtcdStatusSyncPeriod: 2 * time.Second, - FeatureGates: map[featuregate.Feature]bool{ - features.UseEtcdWrapper: true, - }, - EtcdMember: etcd.MemberConfig{ - NotReadyThreshold: 5 * time.Minute, - UnknownThreshold: 1 * time.Minute, - }, - }, assets.CreateImageVector(g)) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(reconciler.RegisterWithManager(mgr)).To(Succeed()) - }) - g.Expect(itTestEnv.StartManager()).To(Succeed()) - t.Log("successfully registered etcd reconciler with manager and started manager") - return ReconcilerTestEnv{ - itTestEnv: itTestEnv, - reconciler: reconciler, - } -} - -func createAndAssertEtcdAndAllManagedResources(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { - g := NewWithT(t) - cl := reconcilerTestEnv.itTestEnv.GetClient() - // create etcdInstance resource - g.Expect(cl.Create(ctx, etcdInstance)).To(Succeed()) - t.Logf("trigggered creation of etcd instance: {name: %s, namespace: %s}, waiting for resources to be created...", etcdInstance.Name, etcdInstance.Namespace) - // ascertain that all etcd resources are created - assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 3*time.Minute, 2*time.Second) - t.Logf("successfully created all resources for etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) - // add finalizer - addFinalizer(ctx, g, cl, client.ObjectKeyFromObject(etcdInstance)) - t.Logf("successfully added finalizer to etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) -} - -func createAndAssertEtcdReconciliation(ctx context.Context, t *testing.T, reconcilerTestEnv ReconcilerTestEnv, etcdInstance *druidv1alpha1.Etcd) { - // create etcd and assert etcd spec reconciliation - createAndAssertEtcdAndAllManagedResources(ctx, t, reconcilerTestEnv, etcdInstance) - - // assert etcd status reconciliation - assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) - expectedLastOperation := &druidv1alpha1.LastOperation{ - Type: druidv1alpha1.LastOperationTypeReconcile, - State: druidv1alpha1.LastOperationStateSucceeded, - } - assertETCDLastOperationAndLastErrorsUpdatedSuccessfully(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), expectedLastOperation, nil, 5*time.Second, 1*time.Second) - assertETCDOperationAnnotation(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), false, 5*time.Second, 1*time.Second) - t.Logf("successfully reconciled status of etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) -} - -func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey client.ObjectKey) { - etcdInstance := &druidv1alpha1.Etcd{} - g.Expect(cl.Get(ctx, etcdObjectKey, etcdInstance)).To(Succeed()) - g.Expect(controllerutils.AddFinalizers(ctx, cl, etcdInstance, common.FinalizerName)).To(Succeed()) -} - -type etcdMemberLeaseConfig struct { - name string - memberID string - role druidv1alpha1.EtcdRole - renewTime *metav1.MicroTime -} - -func updateMemberLeaseSpec(ctx context.Context, t *testing.T, cl client.Client, namespace string, memberLeaseConfigs []etcdMemberLeaseConfig) { - g := NewWithT(t) - for _, config := range memberLeaseConfigs { - lease := &coordinationv1.Lease{} - g.Expect(cl.Get(ctx, client.ObjectKey{Name: config.name, Namespace: namespace}, lease)).To(Succeed()) - updatedLease := lease.DeepCopy() - updatedLease.Spec.HolderIdentity = pointer.String(fmt.Sprintf("%s:%s", config.memberID, config.role)) - updatedLease.Spec.RenewTime = config.renewTime - g.Expect(cl.Update(ctx, updatedLease)).To(Succeed()) - t.Logf("successfully updated member lease %s with holderIdentity: %s", config.name, *updatedLease.Spec.HolderIdentity) - } -} - -func createPVCs(ctx context.Context, t *testing.T, cl client.Client, sts *appsv1.StatefulSet) []*corev1.PersistentVolumeClaim { - g := NewWithT(t) - pvcs := make([]*corev1.PersistentVolumeClaim, 0, int(*sts.Spec.Replicas)) - volClaimName := sts.Spec.VolumeClaimTemplates[0].Name - for i := 0; i < int(*sts.Spec.Replicas); i++ { - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s-%d", volClaimName, sts.Name, i), - Namespace: sts.Namespace, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - } - g.Expect(cl.Create(ctx, pvc)).To(Succeed()) - pvcs = append(pvcs, pvc) - t.Logf("successfully created pvc: %s", pvc.Name) - } - return pvcs -} - -func createPVCWarningEvent(ctx context.Context, t *testing.T, cl client.Client, namespace, pvcName, reason, message string) { - g := NewWithT(t) - event := &corev1.Event{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("%s-event-", pvcName), - Namespace: namespace, - }, - InvolvedObject: corev1.ObjectReference{ - Kind: "PersistentVolumeClaim", - Name: pvcName, - Namespace: namespace, - APIVersion: "v1", - }, - Reason: reason, - Message: message, - Type: corev1.EventTypeWarning, - } - g.Expect(cl.Create(ctx, event)).To(Succeed()) - t.Logf("successfully created warning event for pvc: %s", pvcName) -} From 413e77ae8ecd658398b734fe7a7cf028daa9f210 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 3 Jun 2024 10:59:47 +0530 Subject: [PATCH 218/235] addressed review comments from @ishan16696 --- internal/controller/etcd/register.go | 43 ++++------------------- internal/controller/etcd/register_test.go | 4 +-- 2 files changed, 9 insertions(+), 38 deletions(-) diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index e0c95dd75..2500afdfc 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -9,7 +9,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -42,7 +41,7 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { // Conditions for reconciliation: // Scenario 1: {Auto-Reconcile: false, Reconcile-Annotation-Present: false, Spec-Updated: false/true, Status-Updated: false/true, update-event-reconciled: false} // Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} -// Scenario 3: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} +// Scenario 3: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: true} // Scenario 4: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} // Scenario 5: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} // Scenario 6: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: NA}, This condition cannot happen. In case of a controller restart there will only be a CreateEvent. @@ -52,16 +51,9 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { // Scenario 10: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: false} // Scenario 11: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} // Scenario 12: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} -// Scenario 13: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} +// Scenario 13: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: true} // Scenario 14: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} func (r *Reconciler) buildPredicate() predicate.Predicate { - // If there is no change to spec and status then no reconciliation would happen. This is also true when auto-reconcile - // has been enabled. If an operator wishes to force a reconcile especially when no change (spec/status) has been done to the etcd resource - // then the only way is to explicitly add the reconcile annotation to the etcd resource. - forceReconcilePredicate := predicate.And( - r.hasReconcileAnnotation(), - noSpecAndStatusUpdated(), - ) // If there is a spec change (irrespective of status change) and if there is an update event then it will trigger a reconcile only when either // auto-reconcile has been enabled or an operator has added the reconcile annotation to the etcd resource. onSpecChangePredicate := predicate.And( @@ -73,11 +65,15 @@ func (r *Reconciler) buildPredicate() predicate.Predicate { ) return predicate.Or( - forceReconcilePredicate, + r.hasReconcileAnnotation(), onSpecChangePredicate, ) } +// hasReconcileAnnotation returns a predicate that filters events based on the presence of the reconcile annotation. +// Annotation `gardener.cloud/operation: reconcile` is used to force a reconcile for an etcd resource. Irrespective of +// enablement of `auto-reconcile` this annotation will trigger a reconcile. At the end of a successful reconcile spec flow +// it should be ensured that this annotation is removed successfully. func (r *Reconciler) hasReconcileAnnotation() predicate.Predicate { return predicate.Funcs{ UpdateFunc: func(updateEvent event.UpdateEvent) bool { @@ -117,31 +113,6 @@ func specUpdated() predicate.Predicate { } } -func noSpecAndStatusUpdated() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(updateEvent event.UpdateEvent) bool { - return !hasSpecChanged(updateEvent) && !hasStatusChanged(updateEvent) - }, - GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, - CreateFunc: func(createEvent event.CreateEvent) bool { - return true - }, - DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return true }, - } -} - func hasSpecChanged(updateEvent event.UpdateEvent) bool { return updateEvent.ObjectNew.GetGeneration() != updateEvent.ObjectOld.GetGeneration() } - -func hasStatusChanged(updateEvent event.UpdateEvent) bool { - oldEtcd, ok := updateEvent.ObjectOld.(*druidv1alpha1.Etcd) - if !ok { - return false - } - newEtcd, ok := updateEvent.ObjectNew.(*druidv1alpha1.Etcd) - if !ok { - return false - } - return !apiequality.Semantic.DeepEqual(oldEtcd.Status, newEtcd.Status) -} diff --git a/internal/controller/etcd/register_test.go b/internal/controller/etcd/register_test.go index fa4126f1c..dda981183 100644 --- a/internal/controller/etcd/register_test.go +++ b/internal/controller/etcd/register_test.go @@ -161,7 +161,7 @@ func TestBuildPredicateWithNoAutoReconcileButReconcileAnnotPresent(t *testing.T) shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, - shouldAllowUpdateEvent: false, + shouldAllowUpdateEvent: true, }, { name: "both spec and status have changed", @@ -220,7 +220,7 @@ func TestBuildPredicateWithAutoReconcileAndReconcileAnnotSet(t *testing.T) { shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, - shouldAllowUpdateEvent: false, + shouldAllowUpdateEvent: true, }, { name: "both spec and status have changed", From 0cd42b729e2d89b19828c05c3d4c2f7ca7c782be Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 3 Jun 2024 11:30:09 +0530 Subject: [PATCH 219/235] removed unused code in it tests --- test/it/controller/etcd/helper.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/it/controller/etcd/helper.go b/test/it/controller/etcd/helper.go index ec0321514..e5effa4de 100644 --- a/test/it/controller/etcd/helper.go +++ b/test/it/controller/etcd/helper.go @@ -15,11 +15,9 @@ import ( . "github.com/onsi/gomega" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" "github.com/gardener/etcd-druid/internal/controller/etcd" "github.com/gardener/etcd-druid/internal/features" testutils "github.com/gardener/etcd-druid/test/utils" - "github.com/gardener/gardener/pkg/controllerutils" appsv1 "k8s.io/api/apps/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" @@ -97,10 +95,6 @@ func createAndAssertEtcdAndAllManagedResources(ctx context.Context, t *testing.T t.Logf("successfully updated sts revision for etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) createStsPods(ctx, t, cl, etcdInstance.ObjectMeta, stsUpdateRevision) t.Logf("successfully created pods for statefulset of etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) - - //// add finalizer - //addFinalizer(ctx, g, cl, client.ObjectKeyFromObject(etcdInstance)) - //t.Logf("successfully added finalizer to etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) } func updateAndGetStsRevision(ctx context.Context, t *testing.T, cl client.Client, etcdInstance *druidv1alpha1.Etcd) string { @@ -181,12 +175,6 @@ func createAndAssertEtcdReconciliation(ctx context.Context, t *testing.T, reconc t.Logf("successfully reconciled status of etcd instance: {name: %s, namespace: %s}", etcdInstance.Name, etcdInstance.Namespace) } -func addFinalizer(ctx context.Context, g *WithT, cl client.Client, etcdObjectKey client.ObjectKey) { - etcdInstance := &druidv1alpha1.Etcd{} - g.Expect(cl.Get(ctx, etcdObjectKey, etcdInstance)).To(Succeed()) - g.Expect(controllerutils.AddFinalizers(ctx, cl, etcdInstance, common.FinalizerName)).To(Succeed()) -} - type etcdMemberLeaseConfig struct { name string memberID string From 68b5d1af24378fc1884479a477201171208e5bfe Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 4 Jun 2024 09:20:18 +0530 Subject: [PATCH 220/235] added log for failed status updates --- internal/controller/etcd/reconciler.go | 1 + internal/controller/utils/reconciler.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 991170d2e..1e640e55f 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -100,6 +100,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } if result := r.reconcileStatus(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) { + r.logger.Error(result.GetCombinedError(), "Failed to reconcile status") return result.ReconcileResult() } if reconcileSpecResult.HasErrors() { diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 372b10a23..95907186e 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -64,6 +64,11 @@ func (r ReconcileStepResult) GetErrors() []error { return r.errs } +// GetCombinedError returns the combined error from the reconcile step. +func (r ReconcileStepResult) GetCombinedError() error { + return errors.Join(r.errs...) +} + // GetResult returns the result from the reconcile step. func (r ReconcileStepResult) GetResult() ctrl.Result { return r.result From f6af511415fda5f351b248e1ef1944acf856fa75 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 4 Jun 2024 09:21:08 +0530 Subject: [PATCH 221/235] addressed review comment from @ishan16696 regarding doc string for register.go --- internal/controller/etcd/register.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 2500afdfc..488438568 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -39,20 +39,14 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { // 1. create and delete events are always reconciled irrespective of whether reconcile annotation is present or auto-reconcile has been enabled. // 2. generic events are never reconciled. If there is a need in future to react to generic events then this should be changed. // Conditions for reconciliation: -// Scenario 1: {Auto-Reconcile: false, Reconcile-Annotation-Present: false, Spec-Updated: false/true, Status-Updated: false/true, update-event-reconciled: false} -// Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} -// Scenario 3: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: true} -// Scenario 4: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} -// Scenario 5: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} -// Scenario 6: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: NA}, This condition cannot happen. In case of a controller restart there will only be a CreateEvent. -// Scenario 7: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} -// Scenario 8: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} -// Scenario 9: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} -// Scenario 10: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: false} -// Scenario 11: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: false, update-event-reconciled: true} -// Scenario 12: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} -// Scenario 13: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Status-Updated: true, update-event-reconciled: true} -// Scenario 14: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} +// Scenario 1: {Auto-Reconcile: false, Reconcile-Annotation-Present: false, Spec-Updated: true/false, Status-Updated: true/false, update-event-reconciled: false} +// Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true/false, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 3: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: NA}, This condition cannot happen. In case of a controller restart there will only be a CreateEvent. +// Scenario 4: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} +// Scenario 5: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} +// Scenario 6: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} +// Scenario 7: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: false} +// Scenario 8: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true/false, Status-Updated: true/false, update-event-reconciled: true} func (r *Reconciler) buildPredicate() predicate.Predicate { // If there is a spec change (irrespective of status change) and if there is an update event then it will trigger a reconcile only when either // auto-reconcile has been enabled or an operator has added the reconcile annotation to the etcd resource. From ff88140fcf3f52ee628b2bad7fc131f623fbe18d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 4 Jun 2024 12:26:05 +0530 Subject: [PATCH 222/235] changed client.List to client.Get in GetStatefulSet --- internal/utils/statefulset.go | 19 +++++++++---------- internal/utils/statefulset_test.go | 10 +++++----- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/internal/utils/statefulset.go b/internal/utils/statefulset.go index 46045daf0..9debcac12 100644 --- a/internal/utils/statefulset.go +++ b/internal/utils/statefulset.go @@ -11,11 +11,11 @@ import ( "strings" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -42,17 +42,16 @@ func IsStatefulSetReady(etcdReplicas int32, statefulSet *appsv1.StatefulSet) (bo // GetStatefulSet fetches StatefulSet created for the etcd. func GetStatefulSet(ctx context.Context, cl client.Client, etcd *druidv1alpha1.Etcd) (*appsv1.StatefulSet, error) { - statefulSets := &appsv1.StatefulSetList{} - if err := cl.List(ctx, statefulSets, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: labels.Set(druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta)).AsSelector()}); err != nil { + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKey{Name: druidv1alpha1.GetStatefulSetName(etcd.ObjectMeta), Namespace: etcd.Namespace}, sts); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } return nil, err } - - for _, sts := range statefulSets.Items { - if metav1.IsControlledBy(&sts, etcd) { - return &sts, nil - } + if metav1.IsControlledBy(sts, etcd) { + return sts, nil } - return nil, nil } diff --git a/internal/utils/statefulset_test.go b/internal/utils/statefulset_test.go index 1c9c0aa8a..9bf3127a2 100644 --- a/internal/utils/statefulset_test.go +++ b/internal/utils/statefulset_test.go @@ -11,7 +11,7 @@ import ( "strings" "testing" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/internal/client/kubernetes" testutils "github.com/gardener/etcd-druid/test/utils" . "github.com/onsi/gomega" @@ -128,7 +128,7 @@ func TestGetStatefulSet(t *testing.T) { name string isStsPresent bool ownedByEtcd bool - listErr *apierrors.StatusError + getErr *apierrors.StatusError expectedErr *apierrors.StatusError }{ { @@ -147,10 +147,10 @@ func TestGetStatefulSet(t *testing.T) { ownedByEtcd: true, }, { - name: "returns error when client list fails", + name: "returns error when client get fails", isStsPresent: true, ownedByEtcd: true, - listErr: apiInternalErr, + getErr: apiInternalErr, expectedErr: apiInternalErr, }, } @@ -168,7 +168,7 @@ func TestGetStatefulSet(t *testing.T) { sts := testutils.CreateStatefulSet(etcd.Name, etcd.Namespace, etcdUID, etcd.Spec.Replicas) existingObjects = append(existingObjects, sts) } - cl := testutils.CreateTestFakeClientForAllObjectsInNamespace(nil, tc.listErr, etcd.Namespace, druidv1alpha1.GetDefaultLabels(etcd.ObjectMeta), existingObjects...) + cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.getErr, nil, nil, nil, existingObjects, client.ObjectKey{Name: testutils.TestEtcdName, Namespace: testutils.TestNamespace}) foundSts, err := GetStatefulSet(context.Background(), cl, etcd) if tc.expectedErr != nil { g.Expect(err).To(HaveOccurred()) From e9bee1b06971a9a0865745c6afd81bd6b867aece Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Fri, 7 Jun 2024 14:23:14 +0530 Subject: [PATCH 223/235] refactored sentinel webhook, includes protection for PVC deletion and uses PartialObjectMetadata --- .../templates/validating-webhook-config.yaml | 1 + internal/webhook/sentinel/handler.go | 174 ++++++------------ internal/webhook/sentinel/handler_test.go | 165 +++++------------ internal/webhook/util/decode.go | 133 +++++++++++++ internal/webhook/util/miscellaneous.go | 28 +++ test/utils/manager.go | 45 +++++ test/utils/matcher.go | 2 +- 7 files changed, 314 insertions(+), 234 deletions(-) create mode 100644 internal/webhook/util/decode.go create mode 100644 internal/webhook/util/miscellaneous.go create mode 100644 test/utils/manager.go diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index 0276dacb3..4920a0c35 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -43,6 +43,7 @@ webhooks: - serviceaccounts - services - configmaps + - persistentvolumeclaims scope: '*' - apiGroups: - rbac.authorization.k8s.io diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 650c9eaa0..a6f5ddd9e 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -11,21 +11,15 @@ import ( "slices" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - + "github.com/gardener/etcd-druid/internal/webhook/util" "github.com/go-logr/logr" admissionv1 "k8s.io/api/admission/v1" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/serviceaccount" - "k8s.io/client-go/scale/scheme/autoscalingv1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -35,66 +29,75 @@ var allowedOperations = []admissionv1.Operation{admissionv1.Create, admissionv1. // Handler is the Sentinel Webhook admission handler. type Handler struct { - client.Client + client client.Client config *Config - decoder *admission.Decoder + decoder *util.RequestDecoder logger logr.Logger } // NewHandler creates a new handler for Sentinel Webhook. func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { - decoder := admission.NewDecoder(mgr.GetScheme()) return &Handler{ - Client: mgr.GetClient(), + client: mgr.GetClient(), config: config, - decoder: decoder, + decoder: util.NewRequestDecoder(mgr), logger: mgr.GetLogger().WithName(handlerName), }, nil } // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { - requestGKString := fmt.Sprintf("%s/%s", req.Kind.Group, req.Kind.Kind) + requestGKString := util.GetGroupKindAsStringFromRequest(req) log := h.logger.WithValues("name", req.Name, "namespace", req.Namespace, "resourceGroupKind", requestGKString, "operation", req.Operation, "user", req.UserInfo.Username) log.V(1).Info("Sentinel webhook invoked") - if slices.Contains(allowedOperations, req.Operation) { - return admission.Allowed(fmt.Sprintf("operation %s is allowed", req.Operation)) + if ok, response := h.skipValidationForOperations(req.Operation); ok { + return *response } - etcd, allowedMessage, err := h.getEtcdForRequest(ctx, req) + partialObjMeta, err := h.decoder.DecodeRequestObjectAsPartialObjectMetadata(ctx, req) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if partialObjMeta == nil { + return admission.Allowed(fmt.Sprintf("resource: %v is not supported by Sentinel webhook", requestGKString)) + } + + if !isObjManagedByDruid(partialObjMeta.ObjectMeta) { + return admission.Allowed(fmt.Sprintf("resource: %v is not managed by druid, skipping validations", util.CreateObjectKey(partialObjMeta))) + } + + etcd, warnings, err := h.getParentEtcdObj(ctx, partialObjMeta) if err != nil { return admission.Errored(http.StatusInternalServerError, err) } - if allowedMessage != "" { - return admission.Allowed(allowedMessage) + if etcd == nil { + return admission.Allowed(fmt.Sprintf("resource: %v is not part of any Etcd, skipping validations", util.CreateObjectKey(partialObjMeta))).WithWarnings(warnings...) + } + + // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set. + if !druidv1alpha1.AreManagedResourcesProtected(etcd.ObjectMeta) { + return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) } // allow deletion operation on resources if the Etcd is currently being deleted, but only by etcd-druid and exempt service accounts. if req.Operation == admissionv1.Delete && etcd.IsDeletionInProgress() { - if req.UserInfo.Username == h.config.ReconcilerServiceAccount { - return admission.Allowed(fmt.Sprintf("deletion of resource by etcd-druid is allowed during deletion of Etcd %s", etcd.Name)) - } - if slices.Contains(h.config.ExemptServiceAccounts, req.UserInfo.Username) { - return admission.Allowed(fmt.Sprintf("deletion of resource by exempt SA %s is allowed during deletion of Etcd %s", req.UserInfo.Username, etcd.Name)) - } - return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing deletion of Etcd %s by etcd-druid", etcd.Name)) + return h.handleDelete(req, etcd) } + return h.handleUpdate(req, etcd) +} + +func (h *Handler) handleUpdate(req admission.Request, etcd *druidv1alpha1.Etcd) admission.Response { // Leases (member and snapshot) will be periodically updated by etcd members. // Allow updates to such leases, but only by etcd members, which would use the serviceaccount deployed by druid for them. requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} - if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() && req.Operation == admissionv1.Update { + if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() { if serviceaccount.MatchesUsername(etcd.GetNamespace(), druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), req.UserInfo.Username) { return admission.Allowed("lease resource can be freely updated by etcd members") } } - // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set. - if !druidv1alpha1.AreManagedResourcesProtected(etcd.ObjectMeta) { - return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) - } - // allow operations on resources if the Etcd is currently being reconciled, but only by etcd-druid. if etcd.IsReconciliationInProgress() { if req.UserInfo.Username == h.config.ReconcilerServiceAccount { @@ -113,100 +116,41 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Denied(fmt.Sprintf("changes disallowed, since no ongoing processing of Etcd %s by etcd-druid", etcd.Name)) } -// getEtcdForRequest returns the Etcd resource corresponding to the object in the admission request. -// It also returns admission response message and error, if any. -func (h *Handler) getEtcdForRequest(ctx context.Context, req admission.Request) (*druidv1alpha1.Etcd, string, error) { - obj, err := h.decodeRequestObject(ctx, req) - if err != nil { - return nil, "", err +func (h *Handler) handleDelete(req admission.Request, etcd *druidv1alpha1.Etcd) admission.Response { + if req.UserInfo.Username == h.config.ReconcilerServiceAccount { + return admission.Allowed(fmt.Sprintf("deletion of resource by etcd-druid is allowed during deletion of Etcd %s", etcd.Name)) } - if obj == nil { - return nil, fmt.Sprintf("unexpected resource type: %s/%s", req.Kind.Group, req.Kind.Kind), nil + if slices.Contains(h.config.ExemptServiceAccounts, req.UserInfo.Username) { + return admission.Allowed(fmt.Sprintf("deletion of resource by exempt SA %s is allowed during deletion of Etcd %s", req.UserInfo.Username, etcd.Name)) } + return admission.Denied(fmt.Sprintf("no external intervention allowed during ongoing deletion of Etcd %s by etcd-druid", etcd.Name)) +} - // allow changes for resources not managed by etcd-druid - managedBy, hasLabel := obj.GetLabels()[druidv1alpha1.LabelManagedByKey] - if !hasLabel || managedBy != druidv1alpha1.LabelManagedByValue { - return nil, fmt.Sprintf("resource is not managed by etcd-druid, as label %s is missing", druidv1alpha1.LabelManagedByKey), nil +func (h *Handler) skipValidationForOperations(reqOperation admissionv1.Operation) (bool, *admission.Response) { + skipOperation := slices.Contains(allowedOperations, reqOperation) + skipAllowedResponse := admission.Allowed(fmt.Sprintf("operation %s is allowed", reqOperation)) + if skipOperation { + return true, &skipAllowedResponse } + return false, nil +} - // get the name of the Etcd that the resource is part of - etcdName, hasLabel := obj.GetLabels()[druidv1alpha1.LabelPartOfKey] +func (h *Handler) getParentEtcdObj(ctx context.Context, partialObjMeta *metav1.PartialObjectMetadata) (*druidv1alpha1.Etcd, admission.Warnings, error) { + etcdName, hasLabel := partialObjMeta.GetLabels()[druidv1alpha1.LabelPartOfKey] if !hasLabel { - return nil, fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey), nil + return nil, admission.Warnings{fmt.Sprintf("cannot determine parent etcd resource, label %s not found on resource: %v", druidv1alpha1.LabelPartOfKey, util.CreateObjectKey(partialObjMeta))}, nil } - etcd := &druidv1alpha1.Etcd{} - if err = h.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: req.Namespace}, etcd); err != nil { + if err := h.client.Get(ctx, types.NamespacedName{Name: etcdName, Namespace: partialObjMeta.GetNamespace()}, etcd); err != nil { if apierrors.IsNotFound(err) { - return nil, fmt.Sprintf("corresponding Etcd %s not found", etcdName), nil + return nil, admission.Warnings{fmt.Sprintf("parent Etcd %s not found for resource: %v", etcdName, util.CreateObjectKey(partialObjMeta))}, nil } - return nil, "", err + return nil, nil, err } - - return etcd, "", nil - + return etcd, nil, nil } -// decodeRequestObject decodes the relevant object from the admission request and returns it. -// If it encounters an unexpected resource type, it returns a nil object. -func (h *Handler) decodeRequestObject(ctx context.Context, req admission.Request) (client.Object, error) { - var ( - requestGK = schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} - obj client.Object - err error - ) - switch requestGK { - case - corev1.SchemeGroupVersion.WithKind("ServiceAccount").GroupKind(), - corev1.SchemeGroupVersion.WithKind("Service").GroupKind(), - corev1.SchemeGroupVersion.WithKind("ConfigMap").GroupKind(), - rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(), - rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(), - appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), - policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), - batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), - coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): - obj, err = h.doDecodeRequestObject(req) - - // if admission request is for statefulsets/scale subresource, then fetch and return the parent statefulset object instead. - case autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(): - requestResourceGK := schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} - if requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() { - obj, err = h.fetchStatefulSet(ctx, req.Name, req.Namespace) - } - } - - return obj, err -} - -func (h *Handler) doDecodeRequestObject(req admission.Request) (client.Object, error) { - var ( - err error - obj = &unstructured.Unstructured{} - ) - - if req.Operation == admissionv1.Delete { - if err = h.decoder.DecodeRaw(req.OldObject, obj); err != nil { - return nil, err - } - return obj, nil - } - - if err = h.decoder.Decode(req, obj); err != nil { - return nil, err - } - return obj, nil -} - -func (h *Handler) fetchStatefulSet(ctx context.Context, name, namespace string) (*appsv1.StatefulSet, error) { - var ( - err error - sts = &appsv1.StatefulSet{} - ) - - if err = h.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, sts); err != nil { - return nil, err - } - return sts, nil +func isObjManagedByDruid(objMeta metav1.ObjectMeta) bool { + managedBy, hasLabel := objMeta.GetLabels()[druidv1alpha1.LabelManagedByKey] + return hasLabel && managedBy == druidv1alpha1.LabelManagedByValue } diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/sentinel/handler_test.go index 1583edf7a..93f809687 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/sentinel/handler_test.go @@ -15,7 +15,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/internal/client/kubernetes" testutils "github.com/gardener/etcd-druid/test/utils" - "github.com/go-logr/logr" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admission/v1" @@ -76,16 +75,7 @@ func TestHandleCreateAndConnect(t *testing.T) { } cl := testutils.CreateDefaultFakeClient() - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{Enabled: true}) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -133,16 +123,7 @@ func TestHandleLeaseUpdate(t *testing.T) { t.Run(tc.name, func(t *testing.T) { etcd := testutils.EtcdBuilderWithDefaults(testEtcdName, testNamespace).Build() cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, nil, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{Enabled: true}) obj := buildObjRawExtension(g, &coordinationv1.Lease{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}) @@ -171,7 +152,7 @@ func TestHandleLeaseUpdate(t *testing.T) { } } -func TestHandleStatefulSetScaleSubresourceUpdate(t *testing.T) { +func TestHandleUnmanagedStatefulSetScaleSubresourceUpdate(t *testing.T) { g := NewWithT(t) // create sts without managed-by label @@ -179,16 +160,7 @@ func TestHandleStatefulSetScaleSubresourceUpdate(t *testing.T) { delete(sts.Labels, druidv1alpha1.LabelManagedByKey) cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, nil, nil, nil, nil, []client.Object{sts}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{Enabled: true}) obj := buildObjRawExtension(g, &autoscalingv1.Scale{}, nil, testObjectName, testNamespace, nil) @@ -206,7 +178,7 @@ func TestHandleStatefulSetScaleSubresourceUpdate(t *testing.T) { }) g.Expect(response.Allowed).To(BeTrue()) - g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource is not managed by etcd-druid, as label %s is missing", druidv1alpha1.LabelManagedByKey))) + g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource: %v is not managed by druid, skipping validations", client.ObjectKeyFromObject(sts)))) g.Expect(response.Result.Code).To(Equal(int32(http.StatusOK))) } @@ -214,16 +186,7 @@ func TestUnexpectedResourceType(t *testing.T) { g := NewWithT(t) cl := fake.NewClientBuilder().Build() - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{Enabled: true}) resp := handler.Handle(context.Background(), admission.Request{ AdmissionRequest: admissionv1.AdmissionRequest{ @@ -233,23 +196,14 @@ func TestUnexpectedResourceType(t *testing.T) { }) g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(resp.Result.Message).To(Equal("unexpected resource type: coordination.k8s.io/Unknown")) + g.Expect(resp.Result.Message).To(Equal("resource: coordination.k8s.io/Unknown is not supported by Sentinel webhook")) } func TestMissingManagedByLabel(t *testing.T) { g := NewWithT(t) cl := fake.NewClientBuilder().Build() - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{Enabled: true}) obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelPartOfKey: testEtcdName}) response := handler.Handle(context.Background(), admission.Request{ @@ -265,23 +219,14 @@ func TestMissingManagedByLabel(t *testing.T) { }) g.Expect(response.Allowed).To(Equal(true)) - g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource is not managed by etcd-druid, as label %s is missing", druidv1alpha1.LabelManagedByKey))) + g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource: %v is not managed by druid, skipping validations", client.ObjectKey{Name: testObjectName, Namespace: testNamespace}))) } func TestMissingResourcePartOfLabel(t *testing.T) { g := NewWithT(t) cl := fake.NewClientBuilder().Build() - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{Enabled: true}) obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue}) response := handler.Handle(context.Background(), admission.Request{ @@ -297,7 +242,7 @@ func TestMissingResourcePartOfLabel(t *testing.T) { }) g.Expect(response.Allowed).To(Equal(true)) - g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("label %s not found on resource", druidv1alpha1.LabelPartOfKey))) + g.Expect(response.Result.Message).To(Equal(fmt.Sprintf("resource: %v is not part of any Etcd, skipping validations", client.ObjectKey{Name: testObjectName, Namespace: testNamespace}))) } func TestHandleUpdate(t *testing.T) { @@ -380,18 +325,11 @@ func TestHandleUpdate(t *testing.T) { Build() cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - ReconcilerServiceAccount: reconcilerServiceAccount, - ExemptServiceAccounts: exemptServiceAccounts, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }) obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, tc.objectRaw, testObjectName, testNamespace, tc.objectLabels) response := handler.Handle(context.Background(), admission.Request{ @@ -429,7 +367,7 @@ func TestHandleWithInvalidRequestObject(t *testing.T) { objectRaw: []byte{}, expectedAllowed: false, expectedMessage: "there is no content to decode", - expectedErrorCode: http.StatusInternalServerError, + expectedErrorCode: http.StatusBadRequest, }, { name: "malformed request object", @@ -437,7 +375,7 @@ func TestHandleWithInvalidRequestObject(t *testing.T) { objectRaw: []byte("foo"), expectedAllowed: false, expectedMessage: "invalid character", - expectedErrorCode: http.StatusInternalServerError, + expectedErrorCode: http.StatusBadRequest, }, { name: "empty request object", @@ -445,7 +383,7 @@ func TestHandleWithInvalidRequestObject(t *testing.T) { objectRaw: []byte{}, expectedAllowed: false, expectedMessage: "there is no content to decode", - expectedErrorCode: http.StatusInternalServerError, + expectedErrorCode: http.StatusBadRequest, }, { name: "malformed request object", @@ -453,7 +391,7 @@ func TestHandleWithInvalidRequestObject(t *testing.T) { objectRaw: []byte("foo"), expectedAllowed: false, expectedMessage: "invalid character", - expectedErrorCode: http.StatusInternalServerError, + expectedErrorCode: http.StatusBadRequest, }, } @@ -461,18 +399,11 @@ func TestHandleWithInvalidRequestObject(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cl := testutils.CreateDefaultFakeClient() - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - ReconcilerServiceAccount: reconcilerServiceAccount, - ExemptServiceAccounts: exemptServiceAccounts, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }) obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, tc.objectRaw, testObjectName, testNamespace, nil) @@ -525,18 +456,11 @@ func TestEtcdGetFailures(t *testing.T) { for _, tc := range testCases { t.Run(t.Name(), func(t *testing.T) { cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - ReconcilerServiceAccount: reconcilerServiceAccount, - ExemptServiceAccounts: exemptServiceAccounts, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }) obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, nil, testObjectName, testNamespace, map[string]string{ druidv1alpha1.LabelPartOfKey: testEtcdName, @@ -687,18 +611,11 @@ func TestHandleDelete(t *testing.T) { Build() cl := testutils.CreateTestFakeClientWithSchemeForObjects(kubernetes.Scheme, tc.etcdGetErr, nil, nil, nil, []client.Object{etcd}, client.ObjectKey{Name: testEtcdName, Namespace: testNamespace}) - decoder := admission.NewDecoder(cl.Scheme()) - - handler := &Handler{ - Client: cl, - config: &Config{ - Enabled: true, - ReconcilerServiceAccount: reconcilerServiceAccount, - ExemptServiceAccounts: exemptServiceAccounts, - }, - decoder: decoder, - logger: logr.Discard(), - } + handler := createHandler(g, cl, Config{ + Enabled: true, + ReconcilerServiceAccount: reconcilerServiceAccount, + ExemptServiceAccounts: exemptServiceAccounts, + }) obj := buildObjRawExtension(g, &appsv1.StatefulSet{}, tc.objectRaw, testObjectName, testNamespace, tc.objectLabels) @@ -723,6 +640,18 @@ func TestHandleDelete(t *testing.T) { // ---------------- Helper functions ------------------- +func createHandler(g *WithT, cl client.Client, cfg Config) *Handler { + mgr := &testutils.FakeManager{ + Client: cl, + Scheme: cl.Scheme(), + Logger: logr.Discard(), + } + + h, err := NewHandler(mgr, &cfg) + g.Expect(err).ToNot(HaveOccurred()) + return h +} + func buildObjRawExtension(g *WithT, emptyObj runtime.Object, objRaw []byte, testObjectName, testNs string, labels map[string]string) runtime.RawExtension { var ( rawBytes []byte diff --git a/internal/webhook/util/decode.go b/internal/webhook/util/decode.go new file mode 100644 index 000000000..97531ecf3 --- /dev/null +++ b/internal/webhook/util/decode.go @@ -0,0 +1,133 @@ +package util + +import ( + "context" + "fmt" + + druiderr "github.com/gardener/etcd-druid/internal/errors" + admissionv1 "k8s.io/api/admission/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/scale/scheme/autoscalingv1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + // ErrDecodeRequestObject indicates an error in decoding the request object. + ErrDecodeRequestObject = "ERR_DECODE_REQUEST_OBJECT" + // ErrGetStatefulSet indicates an error in fetching the StatefulSet resource. + ErrGetStatefulSet = "ERR_GET_SCALE_SUBRESOURCE_PARENT" + // ErrTooManyMatchingStatefulSets indicates that more than one StatefulSet was found for the given labels. + ErrTooManyMatchingStatefulSets = "ERR_TOO_MANY_MATCHING_STATEFULSETS" +) + +// RequestDecoder is a decoder for admission requests. +type RequestDecoder struct { + decoder *admission.Decoder + client client.Client +} + +// NewRequestDecoder returns a new RequestDecoder. +func NewRequestDecoder(mgr manager.Manager) *RequestDecoder { + return &RequestDecoder{ + decoder: admission.NewDecoder(mgr.GetScheme()), + client: mgr.GetClient(), + } +} + +// DecodeRequestObjectAsPartialObjectMetadata decodes the request object as a PartialObjectMetadata. +func (d *RequestDecoder) DecodeRequestObjectAsPartialObjectMetadata(ctx context.Context, req admission.Request) (*metav1.PartialObjectMetadata, error) { + var ( + err error + partialObjMeta *metav1.PartialObjectMetadata + ) + requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} + switch requestGK { + case + corev1.SchemeGroupVersion.WithKind("ServiceAccount").GroupKind(), + corev1.SchemeGroupVersion.WithKind("Service").GroupKind(), + corev1.SchemeGroupVersion.WithKind("ConfigMap").GroupKind(), + rbacv1.SchemeGroupVersion.WithKind("Role").GroupKind(), + rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(), + appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), + policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), + batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), + coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): + return d.decodeRequestObjAsPartialObjectMeta(req) + case autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(): + return d.getStatefulSetPartialObjMetaFromScaleSubResource(ctx, req) + case corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim").GroupKind(): + return d.getStatefulSetPartialObjMetaFromPVC(ctx, req) + } + return partialObjMeta, err +} + +func (d *RequestDecoder) decodeRequestObjAsPartialObjectMeta(req admission.Request) (*metav1.PartialObjectMetadata, error) { + var ( + err error + unstructuredObj = &unstructured.Unstructured{} + ) + if req.Operation == admissionv1.Delete { + // OldObject contains the object being deleted + //https://github.com/kubernetes/kubernetes/pull/76346 + err = d.decoder.DecodeRaw(req.OldObject, unstructuredObj) + } else { + err = d.decoder.Decode(req, unstructuredObj) + } + if err != nil { + return nil, druiderr.WrapError(err, + ErrDecodeRequestObject, + "decodeRequestObjectAsPartialObjectMeta", + fmt.Sprintf("failed to decode request object: %v", GetGroupKindFromRequest(req))) + } + return meta.AsPartialObjectMetadata(unstructuredObj), nil +} + +func (d *RequestDecoder) getStatefulSetPartialObjMetaFromScaleSubResource(ctx context.Context, req admission.Request) (*metav1.PartialObjectMetadata, error) { + requestResourceGK := schema.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} + if requestResourceGK == appsv1.SchemeGroupVersion.WithResource("statefulsets").GroupResource() { + objMeta := &metav1.PartialObjectMetadata{} + objMeta.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) + objKey := client.ObjectKey{Name: req.Name, Namespace: req.Namespace} + if err := d.client.Get(ctx, objKey, objMeta); err != nil { + return nil, druiderr.WrapError(err, ErrGetStatefulSet, "FetchStatefulSetByObjectKey", fmt.Sprintf("failed to fetch StatefulSet for objectKey: %v", objKey)) + } + return objMeta, nil + } + return nil, nil +} + +func (d *RequestDecoder) getStatefulSetPartialObjMetaFromPVC(ctx context.Context, req admission.Request) (*metav1.PartialObjectMetadata, error) { + obj, err := d.decodeRequestObjAsPartialObjectMeta(req) + if err != nil { + return nil, err + } + labels := obj.GetLabels() + if len(labels) == 0 { + return nil, fmt.Errorf("resource %s/%s does not have any labels", req.Namespace, req.Name) + } + + objMetaList := &metav1.PartialObjectMetadataList{} + objMetaList.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) + if err = d.client.List(ctx, objMetaList, client.InNamespace(req.Namespace), client.MatchingLabels(labels)); err != nil { + return nil, druiderr.WrapError(err, ErrGetStatefulSet, "FetchStatefulSetByLabels", fmt.Sprintf("failed to fetch StatefulSet for labels: %v", labels)) + } + if len(objMetaList.Items) == 1 { + return nil, &druiderr.DruidError{ + Code: ErrTooManyMatchingStatefulSets, + Operation: "FetchStatefulSetByLabels", + Message: fmt.Sprintf("found more than one StatefulSet for labels: %v", labels), + } + } + return &objMetaList.Items[0], nil +} diff --git a/internal/webhook/util/miscellaneous.go b/internal/webhook/util/miscellaneous.go new file mode 100644 index 000000000..0826dfb9d --- /dev/null +++ b/internal/webhook/util/miscellaneous.go @@ -0,0 +1,28 @@ +package util + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// GetGroupKindAsStringFromRequest returns the GroupKind as a string from the given admission request. +func GetGroupKindAsStringFromRequest(req admission.Request) string { + return fmt.Sprintf("%s/%s", req.Kind.Group, req.Kind.Kind) +} + +// GetGroupKindFromRequest returns the GroupKind from the given admission request. +func GetGroupKindFromRequest(req admission.Request) schema.GroupKind { + return schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} +} + +// CreateObjectKey creates a client.ObjectKey from the given PartialObjectMetadata. +func CreateObjectKey(partialObjMeta *metav1.PartialObjectMetadata) client.ObjectKey { + return client.ObjectKey{ + Namespace: partialObjMeta.Namespace, + Name: partialObjMeta.Name, + } +} diff --git a/test/utils/manager.go b/test/utils/manager.go new file mode 100644 index 000000000..316b3b7fe --- /dev/null +++ b/test/utils/manager.go @@ -0,0 +1,45 @@ +package utils + +import ( + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// FakeManager fakes a manager.Manager. +type FakeManager struct { + manager.Manager + Client client.Client + Cache cache.Cache + EventRecorder record.EventRecorder + APIReader client.Reader + Logger logr.Logger + Scheme *runtime.Scheme +} + +func (f FakeManager) GetClient() client.Client { + return f.Client +} + +func (f FakeManager) GetCache() cache.Cache { + return f.Cache +} + +func (f FakeManager) GetEventRecorderFor(_ string) record.EventRecorder { + return f.EventRecorder +} + +func (f FakeManager) GetAPIReader() client.Reader { + return f.APIReader +} + +func (f FakeManager) GetLogger() logr.Logger { + return f.Logger +} + +func (f FakeManager) GetScheme() *runtime.Scheme { + return f.Scheme +} diff --git a/test/utils/matcher.go b/test/utils/matcher.go index 1e8d5af0e..d61b39229 100644 --- a/test/utils/matcher.go +++ b/test/utils/matcher.go @@ -38,7 +38,7 @@ func (m mapMatcher[T]) Match(actual interface{}) (bool, error) { m.diff = append(m.diff, fmt.Sprintf("expected key: %s to be present", k)) } if v != actualVal { - m.diff = append(m.diff, fmt.Sprintf("expected val: %s for key; %s, found val:%s instead", v, k, actualVal)) + m.diff = append(m.diff, fmt.Sprintf("expected val: %s for key; %v, found val:%v instead", v, k, actualVal)) } } return len(m.diff) == 0, nil From ac9264ecaaefb2b9c075accf3bf27f1ea8847d9d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Sun, 9 Jun 2024 16:40:56 +0530 Subject: [PATCH 224/235] fixed sentinel webhook issues --- .../templates/validating-webhook-config.yaml | 8 ++++++++ internal/errors/errors.go | 8 +++++++- internal/webhook/sentinel/handler.go | 6 +++--- internal/webhook/util/decode.go | 15 +++++++++++---- internal/webhook/util/miscellaneous.go | 14 ++++++++++++++ 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index 4920a0c35..d1d0a0090 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -43,6 +43,14 @@ webhooks: - serviceaccounts - services - configmaps + scope: '*' + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: - persistentvolumeclaims scope: '*' - apiGroups: diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 9c62f78a0..cdd150843 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -35,7 +35,13 @@ type DruidError struct { } func (e *DruidError) Error() string { - return fmt.Sprintf("[Operation: %s, Code: %s] %s", e.Operation, e.Code, e.Cause.Error()) + var msg string + if e.Cause != nil { + msg = e.Cause.Error() + } else { + msg = e.Message + } + return fmt.Sprintf("[Operation: %s, Code: %s] %s", e.Operation, e.Code, msg) } // IsRequeueAfterError checks if the given error is of type DruidError and has the given error code. diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index a6f5ddd9e..8a5f320f1 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -17,7 +17,6 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/serviceaccount" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,7 +56,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R partialObjMeta, err := h.decoder.DecodeRequestObjectAsPartialObjectMetadata(ctx, req) if err != nil { - return admission.Errored(http.StatusBadRequest, err) + return admission.Errored(util.DetermineStatusCode(err), err) } if partialObjMeta == nil { return admission.Allowed(fmt.Sprintf("resource: %v is not supported by Sentinel webhook", requestGKString)) @@ -89,9 +88,10 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R } func (h *Handler) handleUpdate(req admission.Request, etcd *druidv1alpha1.Etcd) admission.Response { + requestGK := util.GetGroupKindFromRequest(req) + // Leases (member and snapshot) will be periodically updated by etcd members. // Allow updates to such leases, but only by etcd members, which would use the serviceaccount deployed by druid for them. - requestGK := schema.GroupKind{Group: req.Kind.Group, Kind: req.Kind.Kind} if requestGK == coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind() { if serviceaccount.MatchesUsername(etcd.GetNamespace(), druidv1alpha1.GetServiceAccountName(etcd.ObjectMeta), req.UserInfo.Username) { return admission.Allowed("lease resource can be freely updated by etcd members") diff --git a/internal/webhook/util/decode.go b/internal/webhook/util/decode.go index 97531ecf3..5524b3779 100644 --- a/internal/webhook/util/decode.go +++ b/internal/webhook/util/decode.go @@ -100,7 +100,7 @@ func (d *RequestDecoder) getStatefulSetPartialObjMetaFromScaleSubResource(ctx co objMeta.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) objKey := client.ObjectKey{Name: req.Name, Namespace: req.Namespace} if err := d.client.Get(ctx, objKey, objMeta); err != nil { - return nil, druiderr.WrapError(err, ErrGetStatefulSet, "FetchStatefulSetByObjectKey", fmt.Sprintf("failed to fetch StatefulSet for objectKey: %v", objKey)) + return nil, druiderr.WrapError(err, ErrGetStatefulSet, "GetStatefulSetPartialObjectMetaFromScaleSubResource", fmt.Sprintf("failed to fetch StatefulSet for scale subresource: %v", objKey)) } return objMeta, nil } @@ -114,20 +114,27 @@ func (d *RequestDecoder) getStatefulSetPartialObjMetaFromPVC(ctx context.Context } labels := obj.GetLabels() if len(labels) == 0 { - return nil, fmt.Errorf("resource %s/%s does not have any labels", req.Namespace, req.Name) + return nil, &druiderr.DruidError{ + Code: ErrGetStatefulSet, + Operation: "GetStatefulSetPartialObjMetaFromPVC", + Message: fmt.Sprintf("resource %s/%s does not have any labels", req.Namespace, req.Name), + } } objMetaList := &metav1.PartialObjectMetadataList{} objMetaList.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet")) if err = d.client.List(ctx, objMetaList, client.InNamespace(req.Namespace), client.MatchingLabels(labels)); err != nil { - return nil, druiderr.WrapError(err, ErrGetStatefulSet, "FetchStatefulSetByLabels", fmt.Sprintf("failed to fetch StatefulSet for labels: %v", labels)) + return nil, druiderr.WrapError(err, ErrGetStatefulSet, "GetStatefulSetPartialObjMetaFromPVC", fmt.Sprintf("failed to fetch StatefulSet for labels: %v", labels)) } - if len(objMetaList.Items) == 1 { + if len(objMetaList.Items) > 1 { return nil, &druiderr.DruidError{ Code: ErrTooManyMatchingStatefulSets, Operation: "FetchStatefulSetByLabels", Message: fmt.Sprintf("found more than one StatefulSet for labels: %v", labels), } } + if len(objMetaList.Items) == 0 { + return nil, nil + } return &objMetaList.Items[0], nil } diff --git a/internal/webhook/util/miscellaneous.go b/internal/webhook/util/miscellaneous.go index 0826dfb9d..71f16682e 100644 --- a/internal/webhook/util/miscellaneous.go +++ b/internal/webhook/util/miscellaneous.go @@ -1,8 +1,11 @@ package util import ( + "errors" "fmt" + "net/http" + druiderr "github.com/gardener/etcd-druid/internal/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,3 +29,14 @@ func CreateObjectKey(partialObjMeta *metav1.PartialObjectMetadata) client.Object Name: partialObjMeta.Name, } } + +// DetermineStatusCode determines the HTTP status code based on the given error. +func DetermineStatusCode(err error) int32 { + var druidErr *druiderr.DruidError + if errors.As(err, &druidErr) { + if druidErr.Code == ErrDecodeRequestObject { + return http.StatusBadRequest + } + } + return http.StatusInternalServerError +} From 78411aee114e7abb6312d8bcb730e02926639e82 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 10 Jun 2024 14:30:17 +0530 Subject: [PATCH 225/235] refactored decoder utility for webhook and added a utility method to druiderr --- internal/errors/errors.go | 6 +++ internal/webhook/sentinel/handler.go | 8 ++++ internal/webhook/util/decode.go | 57 ++++++++++++++++++++-------- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/internal/errors/errors.go b/internal/errors/errors.go index cdd150843..462fa445e 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -44,6 +44,12 @@ func (e *DruidError) Error() string { return fmt.Sprintf("[Operation: %s, Code: %s] %s", e.Operation, e.Code, msg) } +// WithCause sets the underline error and returns the DruidError. +func (e *DruidError) WithCause(err error) error { + e.Cause = err + return e +} + // IsRequeueAfterError checks if the given error is of type DruidError and has the given error code. func IsRequeueAfterError(err error) bool { druidErr := &DruidError{} diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/sentinel/handler.go index 8a5f320f1..8e62f24ad 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/sentinel/handler.go @@ -44,6 +44,14 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { }, nil } +/* + handler.Handle -> + 1. check if operation is handled + 2. get object or partial object metadata + 3. pre-checks on object before handling operation + 4. handle operation +*/ + // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { requestGKString := util.GetGroupKindAsStringFromRequest(req) diff --git a/internal/webhook/util/decode.go b/internal/webhook/util/decode.go index 5524b3779..a84c260d1 100644 --- a/internal/webhook/util/decode.go +++ b/internal/webhook/util/decode.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/scale/scheme/autoscalingv1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -63,7 +64,7 @@ func (d *RequestDecoder) DecodeRequestObjectAsPartialObjectMetadata(ctx context. policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): - return d.decodeRequestObjAsPartialObjectMeta(req) + return d.doDecodeRequestObjAsPartialObjectMeta(req) case autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(): return d.getStatefulSetPartialObjMetaFromScaleSubResource(ctx, req) case corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim").GroupKind(): @@ -72,25 +73,49 @@ func (d *RequestDecoder) DecodeRequestObjectAsPartialObjectMetadata(ctx context. return partialObjMeta, err } -func (d *RequestDecoder) decodeRequestObjAsPartialObjectMeta(req admission.Request) (*metav1.PartialObjectMetadata, error) { - var ( - err error - unstructuredObj = &unstructured.Unstructured{} - ) +func (d *RequestDecoder) doDecodeRequestObjAsPartialObjectMeta(req admission.Request) (*metav1.PartialObjectMetadata, error) { + oldObj, newObj, err := d.doDecodeRequestObjects(req) + if err != nil { + return nil, err + } if req.Operation == admissionv1.Delete { + return meta.AsPartialObjectMetadata(oldObj), err + } + return meta.AsPartialObjectMetadata(newObj), err +} + +func (d *RequestDecoder) doDecodeRequestObjects(req admission.Request) (oldObj, newObj *unstructured.Unstructured, err error) { + gk := GetGroupKindFromRequest(req) + switch req.Operation { + case admissionv1.Connect: + return + case admissionv1.Create: + newObj, err = d.decodeObjectAsUnstructured(gk, req.Object) + return + case admissionv1.Delete: // OldObject contains the object being deleted //https://github.com/kubernetes/kubernetes/pull/76346 - err = d.decoder.DecodeRaw(req.OldObject, unstructuredObj) - } else { - err = d.decoder.Decode(req, unstructuredObj) + oldObj, err = d.decodeObjectAsUnstructured(gk, req.OldObject) + return + case admissionv1.Update: + if newObj, err = d.decodeObjectAsUnstructured(gk, req.Object); err != nil { + return + } + if oldObj, err = d.decodeObjectAsUnstructured(gk, req.OldObject); err != nil { + return + } + default: + err = druiderr.WrapError(fmt.Errorf("unsupported operation %s", req.Operation), ErrDecodeRequestObject, "doDecodeRequestObjects", "unsupported operation") } - if err != nil { - return nil, druiderr.WrapError(err, - ErrDecodeRequestObject, - "decodeRequestObjectAsPartialObjectMeta", - fmt.Sprintf("failed to decode request object: %v", GetGroupKindFromRequest(req))) + return +} + +func (d *RequestDecoder) decodeObjectAsUnstructured(gk schema.GroupKind, rawObj runtime.RawExtension) (*unstructured.Unstructured, error) { + obj := &unstructured.Unstructured{} + if err := d.decoder.DecodeRaw(rawObj, obj); err != nil { + return nil, druiderr.WrapError(err, ErrDecodeRequestObject, "decodeObjectAsUnstructured", fmt.Sprintf("failed to decode object: %v", gk)) } - return meta.AsPartialObjectMetadata(unstructuredObj), nil + return obj, nil } func (d *RequestDecoder) getStatefulSetPartialObjMetaFromScaleSubResource(ctx context.Context, req admission.Request) (*metav1.PartialObjectMetadata, error) { @@ -108,7 +133,7 @@ func (d *RequestDecoder) getStatefulSetPartialObjMetaFromScaleSubResource(ctx co } func (d *RequestDecoder) getStatefulSetPartialObjMetaFromPVC(ctx context.Context, req admission.Request) (*metav1.PartialObjectMetadata, error) { - obj, err := d.decodeRequestObjAsPartialObjectMeta(req) + obj, err := d.doDecodeRequestObjAsPartialObjectMeta(req) if err != nil { return nil, err } From 2406a6e2d757838131669b8a1cb5f5d3d1b5d41d Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Tue, 11 Jun 2024 13:10:26 +0530 Subject: [PATCH 226/235] renamed sentinel webhook to etcdcompents and adapted cli, env vars and annotation names.removed jobs from webhook allowing compaction to delete them --- api/v1alpha1/constants.go | 5 +- api/v1alpha1/helper.go | 2 +- api/v1alpha1/helper_test.go | 6 +- .../templates/controller-deployment.yaml | 8 +-- .../templates/validating-webhook-config.yaml | 10 ++-- charts/druid/values.yaml | 2 +- docs/concepts/webhooks.md | 16 ++--- docs/deployment/cli-flags.md | 60 +++++++++---------- docs/development/getting-started-locally.md | 2 +- hack/ci-e2e-kind.sh | 2 +- internal/webhook/config.go | 12 ++-- .../{sentinel => etcdcomponents}/config.go | 36 +++++------ .../{sentinel => etcdcomponents}/handler.go | 24 +++----- .../handler_test.go | 16 ++--- .../{sentinel => etcdcomponents}/register.go | 6 +- internal/webhook/register.go | 14 ++--- internal/webhook/util/decode.go | 2 - skaffold.yaml | 6 +- 18 files changed, 107 insertions(+), 122 deletions(-) rename internal/webhook/{sentinel => etcdcomponents}/config.go (63%) rename internal/webhook/{sentinel => etcdcomponents}/handler.go (91%) rename internal/webhook/{sentinel => etcdcomponents}/handler_test.go (97%) rename internal/webhook/{sentinel => etcdcomponents}/register.go (86%) diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go index 62ce975ff..87f2daf57 100644 --- a/api/v1alpha1/constants.go +++ b/api/v1alpha1/constants.go @@ -25,6 +25,7 @@ const ( IgnoreReconciliationAnnotation = "druid.gardener.cloud/ignore-reconciliation" // SuspendEtcdSpecReconcileAnnotation is an annotation set by an operator to temporarily suspend any etcd spec reconciliation. SuspendEtcdSpecReconcileAnnotation = "druid.gardener.cloud/suspend-etcd-spec-reconcile" - // DisableResourceProtectionAnnotation is an annotation set by an operator to disable protection of resources managed by etcd-druid. - DisableResourceProtectionAnnotation = "druid.gardener.cloud/disable-resource-protection" + // DisableEtcdComponentProtectionAnnotation is an annotation set by an operator to disable protection of components created for + // an etcd cluster and managed by etcd-druid. + DisableEtcdComponentProtectionAnnotation = "druid.gardener.cloud/disable-etcd-component-protection" ) diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index 6d95826d9..e9468b44a 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -108,7 +108,7 @@ func GetSuspendEtcdSpecReconcileAnnotationKey(etcdObjMeta metav1.ObjectMeta) *st // AreManagedResourcesProtected returns false if the Etcd resource has the `druid.gardener.cloud/disable-resource-protection` annotation set, // else returns true. func AreManagedResourcesProtected(etcdObjMeta metav1.ObjectMeta) bool { - return !metav1.HasAnnotation(etcdObjMeta, DisableResourceProtectionAnnotation) + return !metav1.HasAnnotation(etcdObjMeta, DisableEtcdComponentProtectionAnnotation) } // GetDefaultLabels returns the default labels for etcd. diff --git a/api/v1alpha1/helper_test.go b/api/v1alpha1/helper_test.go index 4efad097b..ec28dfc7f 100644 --- a/api/v1alpha1/helper_test.go +++ b/api/v1alpha1/helper_test.go @@ -158,13 +158,13 @@ func TestAreManagedResourcesProtected(t *testing.T) { expectedResourceProtection bool }{ { - name: "No DisableResourceProtectionAnnotation annotation is set", + name: "No DisableEtcdComponentProtectionAnnotation annotation is set", annotations: nil, expectedResourceProtection: true, }, { - name: "DisableResourceProtectionAnnotation is set", - annotations: map[string]string{DisableResourceProtectionAnnotation: ""}, + name: "DisableEtcdComponentProtectionAnnotation is set", + annotations: map[string]string{DisableEtcdComponentProtectionAnnotation: ""}, expectedResourceProtection: false, }, } diff --git a/charts/druid/templates/controller-deployment.yaml b/charts/druid/templates/controller-deployment.yaml index dc413d4ba..09435dbf3 100644 --- a/charts/druid/templates/controller-deployment.yaml +++ b/charts/druid/templates/controller-deployment.yaml @@ -82,11 +82,11 @@ spec: - --secret-workers={{ .Values.controllers.secret.workers }} {{- end }} - {{- if ((.Values.webhooks).sentinel).enabled }} - - --enable-sentinel-webhook=true + {{- if ((.Values.webhooks).etcdcomponents).enabled }} + - --enable-etcd-components-webhook=true - --reconciler-service-account=system:serviceaccount:{{ .Release.Namespace }}:etcd-druid - {{- if .Values.webhooks.sentinel.exemptServiceAccounts }} - - --sentinel-exempt-service-accounts={{ join "," .Values.webhooks.sentinel.exemptServiceAccounts }} + {{- if .Values.webhooks.etcdcomponents.exemptServiceAccounts }} + - --etcd-components-webhook-exempt-service-accounts={{ join "," .Values.webhooks.etcdcomponents.exemptServiceAccounts }} {{- end }} {{- end }} diff --git a/charts/druid/templates/validating-webhook-config.yaml b/charts/druid/templates/validating-webhook-config.yaml index d1d0a0090..9009a2fc3 100644 --- a/charts/druid/templates/validating-webhook-config.yaml +++ b/charts/druid/templates/validating-webhook-config.yaml @@ -13,7 +13,7 @@ metadata: labels: app.kubernetes.io/name: etcd-druid webhooks: - {{- if ((.Values.webhooks).sentinel).enabled }} + {{- if ((.Values.webhooks).etcdcomponents).enabled }} - admissionReviewVersions: - v1beta1 - v1 @@ -22,11 +22,11 @@ webhooks: service: name: etcd-druid namespace: {{ .Release.Namespace }} - path: /webhooks/sentinel + path: /webhooks/etcdcomponents port: {{ .Values.controllerManager.server.webhook.port }} failurePolicy: Fail matchPolicy: Exact - name: sentinel.webhooks.druid.gardener.cloud + name: etcdcomponents.webhooks.druid.gardener.cloud namespaceSelector: {} objectSelector: matchLabels: @@ -118,11 +118,11 @@ webhooks: service: name: etcd-druid namespace: {{ .Release.Namespace }} - path: /webhooks/sentinel + path: /webhooks/etcdcomponents port: {{ .Values.controllerManager.server.webhook.port }} failurePolicy: Fail matchPolicy: Exact - name: stsscale.sentinel.webhooks.druid.gardener.cloud + name: stsscale.etcdcomponents.webhooks.druid.gardener.cloud namespaceSelector: {} rules: - apiGroups: diff --git a/charts/druid/values.yaml b/charts/druid/values.yaml index 21c93ee70..f73069e92 100644 --- a/charts/druid/values.yaml +++ b/charts/druid/values.yaml @@ -49,7 +49,7 @@ controllers: workers: 10 webhooks: - sentinel: + etcdcomponents: enabled: false # reconciler-service-account: system:serviceaccount:{{ .Release.Namespace }}:etcd-druid exemptServiceAccounts: diff --git a/docs/concepts/webhooks.md b/docs/concepts/webhooks.md index fb6afa4e9..0d0d100d0 100644 --- a/docs/concepts/webhooks.md +++ b/docs/concepts/webhooks.md @@ -6,10 +6,10 @@ All webhooks that are a part of etcd-druid reside in package `internal/webhook`, ## Package Structure -The typical package structure for the webhooks that are part of etcd-druid is shown with the *sentinel webhook*: +The typical package structure for the webhooks that are part of etcd-druid is shown with the *EtcdComponents Webhook*: ``` bash -internal/webhook/sentinel +internal/webhook/etcdcomponents ├── config.go ├── handler.go └── register.go @@ -21,16 +21,16 @@ internal/webhook/sentinel Each webhook package may also contain auxiliary files which are relevant to that specific webhook. -## Sentinel Webhook +## Etcd Components Webhook -Druid controller-manager registers and runs the [etcd controller](controllers.md#etcd-controller), which creates and manages various resources such as `Leases`, `ConfigMap`s, and the `Statefulset` for the etcd cluster. It is essential for all these resources to contain correct configuration for the proper functioning of the etcd cluster. +Druid controller-manager registers and runs the [etcd controller](controllers.md#etcd-controller), which creates and manages various components/resources such as `Leases`, `ConfigMap`s, and the `Statefulset` for the etcd cluster. It is essential for all these resources to contain correct configuration for the proper functioning of the etcd cluster. -Unintended changes to any of these *managed resources* can lead to misconfiguration of the etcd cluster, leading to unwanted downtime for etcd traffic. To prevent such unintended changes, a validating webhook called *sentinel webhook* guards these managed resources, ensuring that only authorized entities can perform operations on these managed resources. +Unintended changes to any of these *managed resources* can lead to misconfiguration of the etcd cluster, leading to unwanted downtime for etcd traffic. To prevent such unintended changes, a validating webhook called *EtcdComponents Webhook* guards these managed resources, ensuring that only authorized entities can perform operations on these managed resources. -*Sentinel webhook* prevents *UPDATE* and *DELETE* operations on all resources managed by *etcd controller*, unless such an operation is performed by druid itself, and during reconciliation of the `Etcd` resource. Operations are also allowed if performed by one of the authorized entities specified by CLI flag `--sentinel-exempt-service-accounts`, but only if the `Etcd` resource is not being reconciled by etcd-druid at that time. +*EtcdComponents webhook* prevents *UPDATE* and *DELETE* operations on all resources managed by *etcd controller*, unless such an operation is performed by druid itself, and during reconciliation of the `Etcd` resource. Operations are also allowed if performed by one of the authorized entities specified by CLI flag `--etcd-components-webhook-exempt-service-accounts`, but only if the `Etcd` resource is not being reconciled by etcd-druid at that time. -There may be specific cases where a human operator may need to make changes to the managed resources, possibly to test or fix an etcd cluster. An example of this is [recovery from permanent quorum loss](../operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md), where a human operator will need to suspend reconciliation of the `Etcd` resource, make changes to the underlying managed resources such as `StatefulSet` and `ConfigMap`, and then resume reconciliation for the `Etcd` resource. Such manual interventions will require out-of-band changes to the managed resources. Protection of managed resources for such `Etcd` resources can be turned off by adding an annotation `druid.gardener.cloud/disable-resource-protection` on the `Etcd` resource. This will effectively disable *sentinel webhook* protection for all managed resources for the specific `Etcd`. +There may be specific cases where a human operator may need to make changes to the managed resources, possibly to test or fix an etcd cluster. An example of this is [recovery from permanent quorum loss](../operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md), where a human operator will need to suspend reconciliation of the `Etcd` resource, make changes to the underlying managed resources such as `StatefulSet` and `ConfigMap`, and then resume reconciliation for the `Etcd` resource. Such manual interventions will require out-of-band changes to the managed resources. Protection of managed resources for such `Etcd` resources can be turned off by adding an annotation `druid.gardener.cloud/disable-etcd-component-protection` on the `Etcd` resource. This will effectively disable *EtcdComponents Webhook* protection for all managed resources for the specific `Etcd`. **Note:** *UPDATE* operations for `Lease`s by etcd members are always allowed, since these are regularly updated by the etcd-backup-restore sidecar. -The *sentinel webhook* is disabled by default, and can be enabled via the CLI flag `--enable-sentinel-webhook`. +The *Etcd Components Webhook* is disabled by default, and can be enabled via the CLI flag `--enable-etcd-components-webhook. diff --git a/docs/deployment/cli-flags.md b/docs/deployment/cli-flags.md index 2da5546c0..ded9ed714 100644 --- a/docs/deployment/cli-flags.md +++ b/docs/deployment/cli-flags.md @@ -2,33 +2,33 @@ Etcd-druid exposes the following CLI flags that allow for configuring its behavior. -| CLI FLag | Component | Description | Default | -|-----------------------------------------|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------| -| `feature-gates` | `etcd-druid` | A set of key=value pairs that describe feature gates for alpha/experimental features. Please check [feature-gates](feature-gates.md) for more information. | `""` | -| `metrics-bind-address` | `controller-manager` | The IP address that the metrics endpoint binds to. | `""` | -| `metrics-port` | `controller-manager` | The port used for the metrics endpoint. | `8080` | -| `metrics-addr` | `controller-manager` | The fully qualified address:port that the metrics endpoint binds to.
Deprecated: this field will be eventually removed. Please use `--metrics-bind-address` and --`metrics-port` instead. | `":8080"` | -| `webhook-server-bind-address` | `controller-manager` | The IP address on which to listen for the HTTPS webhook server. | `""` | -| `webhook-server-port` | `controller-manager` | The port on which to listen for the HTTPS webhook server. | `9443` | -| `webhook-server-tls-server-cert-dir` | `controller-manager` | The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively). | `"/etc/webhook-server-tls"` | -| `enable-leader-election` | `controller-manager` | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. | `false` | -| `leader-election-id` | `controller-manager` | Name of the resource that leader election will use for holding the leader lock. | `"druid-leader-election"` | -| `leader-election-resource-lock` | `controller-manager` | Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.
Deprecated. Will be removed in the future in favour of using only `leases` as the leader election resource lock for the controller manager. | `"leases"` | -| `disable-lease-cache` | `controller-manager` | Disable cache for lease.coordination.k8s.io resources. | `false` | -| `etcd-workers` | `etcd-controller` | Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed. | `3` | -| `ignore-operation-annotation` | `etcd-controller` | Specifies whether to ignore or honour the annotation `gardener.cloud/operation: reconcile` on resources to be reconciled.
Deprecated: please use `--enable-etcd-spec-auto-reconcile` instead. | `false` | -| `enable-etcd-spec-auto-reconcile` | `etcd-controller` | If true then automatically reconciles Etcd Spec. If false, waits for explicit annotation `gardener.cloud/operation: reconcile` to be placed on the Etcd resource to trigger reconcile. | `false` | -| `disable-etcd-serviceaccount-automount` | `etcd-controller` | If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets. | `false` | -| `etcd-status-sync-period` | `etcd-controller` | Period after which an etcd status sync will be attempted. | `15s` | -| `etcd-member-notready-threshold` | `etcd-controller` | Threshold after which an etcd member is considered not ready if the status was unknown before. | `5m` | -| `etcd-member-unknown-threshold` | `etcd-controller` | Threshold after which an etcd member is considered unknown. | `1m` | -| `enable-backup-compaction` | `compaction-controller` | Enable automatic compaction of etcd backups. | `false` | -| `compaction-workers` | `compaction-controller` | Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. If compaction is enabled, the value for this flag must be greater than zero. | `3` | -| `etcd-events-threshold` | `compaction-controller` | Total number of etcd events that can be allowed before a backup compaction job is triggered. | `1000000` | -| `active-deadline-duration` | `compaction-controller` | Duration after which a running backup compaction job will be terminated. | `3h` | -| `metrics-scrape-wait-duration` | `compaction-controller` | Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped. | `0s` | -| `etcd-copy-backups-task-workers` | `etcdcopybackupstask-controller` | Number of worker threads for the etcdcopybackupstask controller. | `3` | -| `secret-workers` | `secret-controller` | Number of worker threads for the secrets controller. | `10` | -| `enable-sentinel-webhook` | `sentinel-webhook` | Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid. | `false` | -| `reconciler-service-account` | `sentinel-webhook` | The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. If unspecified, the default service account mounted for etcd-druid will be used. | `` | -| `sentinel-exempt-service-accounts` | `sentinel-webhook` | The comma-separated list of fully qualified names of service accounts that are exempt from Sentinel Webhook checks. | `""` | +| CLI FLag | Component | Description | Default | +|-------------------------------------------|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------| +| `feature-gates` | `etcd-druid` | A set of key=value pairs that describe feature gates for alpha/experimental features. Please check [feature-gates](feature-gates.md) for more information. | `""` | +| `metrics-bind-address` | `controller-manager` | The IP address that the metrics endpoint binds to. | `""` | +| `metrics-port` | `controller-manager` | The port used for the metrics endpoint. | `8080` | +| `metrics-addr` | `controller-manager` | The fully qualified address:port that the metrics endpoint binds to.
Deprecated: this field will be eventually removed. Please use `--metrics-bind-address` and --`metrics-port` instead. | `":8080"` | +| `webhook-server-bind-address` | `controller-manager` | The IP address on which to listen for the HTTPS webhook server. | `""` | +| `webhook-server-port` | `controller-manager` | The port on which to listen for the HTTPS webhook server. | `9443` | +| `webhook-server-tls-server-cert-dir` | `controller-manager` | The path to a directory containing the server's TLS certificate and key (the files must be named tls.crt and tls.key respectively). | `"/etc/webhook-server-tls"` | +| `enable-leader-election` | `controller-manager` | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. | `false` | +| `leader-election-id` | `controller-manager` | Name of the resource that leader election will use for holding the leader lock. | `"druid-leader-election"` | +| `leader-election-resource-lock` | `controller-manager` | Specifies which resource type to use for leader election. Supported options are 'endpoints', 'configmaps', 'leases', 'endpointsleases' and 'configmapsleases'.
Deprecated. Will be removed in the future in favour of using only `leases` as the leader election resource lock for the controller manager. | `"leases"` | +| `disable-lease-cache` | `controller-manager` | Disable cache for lease.coordination.k8s.io resources. | `false` | +| `etcd-workers` | `etcd-controller` | Number of workers spawned for concurrent reconciles of etcd spec and status changes. If not specified then default of 3 is assumed. | `3` | +| `ignore-operation-annotation` | `etcd-controller` | Specifies whether to ignore or honour the annotation `gardener.cloud/operation: reconcile` on resources to be reconciled.
Deprecated: please use `--enable-etcd-spec-auto-reconcile` instead. | `false` | +| `enable-etcd-spec-auto-reconcile` | `etcd-controller` | If true then automatically reconciles Etcd Spec. If false, waits for explicit annotation `gardener.cloud/operation: reconcile` to be placed on the Etcd resource to trigger reconcile. | `false` | +| `disable-etcd-serviceaccount-automount` | `etcd-controller` | If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd StatefulSets. | `false` | +| `etcd-status-sync-period` | `etcd-controller` | Period after which an etcd status sync will be attempted. | `15s` | +| `etcd-member-notready-threshold` | `etcd-controller` | Threshold after which an etcd member is considered not ready if the status was unknown before. | `5m` | +| `etcd-member-unknown-threshold` | `etcd-controller` | Threshold after which an etcd member is considered unknown. | `1m` | +| `enable-backup-compaction` | `compaction-controller` | Enable automatic compaction of etcd backups. | `false` | +| `compaction-workers` | `compaction-controller` | Number of worker threads of the CompactionJob controller. The controller creates a backup compaction job if a certain etcd event threshold is reached. If compaction is enabled, the value for this flag must be greater than zero. | `3` | +| `etcd-events-threshold` | `compaction-controller` | Total number of etcd events that can be allowed before a backup compaction job is triggered. | `1000000` | +| `active-deadline-duration` | `compaction-controller` | Duration after which a running backup compaction job will be terminated. | `3h` | +| `metrics-scrape-wait-duration` | `compaction-controller` | Duration to wait for after compaction job is completed, to allow Prometheus metrics to be scraped. | `0s` | +| `etcd-copy-backups-task-workers` | `etcdcopybackupstask-controller` | Number of worker threads for the etcdcopybackupstask controller. | `3` | +| `secret-workers` | `secret-controller` | Number of worker threads for the secrets controller. | `10` | +| `enable-etcd-components-webhook` | `etcdcomponents-webhook` | Enable EtcdComponents Webhook to prevent unintended changes to resources managed by etcd-druid. | `false` | +| `reconciler-service-account` | `etcdcomponents-webhook` | The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. If unspecified, the default service account mounted for etcd-druid will be used. | `` | +| `etcd-components-exempt-service-accounts` | `etcdcomponents-webhook` | The comma-separated list of fully qualified names of service accounts that are exempt from EtcdComponents Webhook checks. | `""` | diff --git a/docs/development/getting-started-locally.md b/docs/development/getting-started-locally.md index 3ec22c314..1af9cc944 100644 --- a/docs/development/getting-started-locally.md +++ b/docs/development/getting-started-locally.md @@ -56,7 +56,7 @@ Either one of these commands may be used to deploy etcd-druid to the configured This generates the `Etcd` and `EtcdCopyBackupsTask` CRDs and deploys an etcd-druid pod into the cluster. > **Note:** Before calling any of the `make deploy*` commands, certain environment variables may be set in order to enable/disable certain functionalities of etcd-druid. These are: -> - `DRUID_ENABLE_SENTINEL_WEBHOOK=true` : enables the [sentinel webhook](../concepts/webhooks.md#sentinel-webhook) +> - `DRUID_ENABLE_ETCD_COMPONENTS_WEBHOOK=true` : enables the [etcdcomponents webhook](../concepts/webhooks.md#etcd-components-webhook) > - `DRUID_E2E_TEST=true` : sets specific configuration for etcd-druid for optimal e2e test runs, like a lower sync period for the etcd controller. > - `USE_ETCD_DRUID_FEATURE_GATES=false` : enables etcd-druid feature gates. diff --git a/hack/ci-e2e-kind.sh b/hack/ci-e2e-kind.sh index f4acb6bc1..d6ee20087 100755 --- a/hack/ci-e2e-kind.sh +++ b/hack/ci-e2e-kind.sh @@ -25,6 +25,6 @@ make LOCALSTACK_HOST="localstack.default:4566" \ AWS_REGION="us-east-2" \ PROVIDERS="aws" \ TEST_ID="$BUCKET_NAME" \ - DRUID_ENABLE_SENTINEL_WEBHOOK=true \ + DRUID_ENABLE_ETCD_COMPONENTS_WEBHOOK=true \ STEPS="setup,deploy,test" \ test-e2e \ No newline at end of file diff --git a/internal/webhook/config.go b/internal/webhook/config.go index 78b0a24a0..31208c5fb 100644 --- a/internal/webhook/config.go +++ b/internal/webhook/config.go @@ -5,25 +5,25 @@ package webhook import ( - "github.com/gardener/etcd-druid/internal/webhook/sentinel" + "github.com/gardener/etcd-druid/internal/webhook/etcdcomponents" flag "github.com/spf13/pflag" ) // Config defines the configuration for etcd-druid webhooks. type Config struct { - // Sentinel is the configuration required for sentinel webhook. - Sentinel *sentinel.Config + // EtcdComponents is the configuration required for etcdcomponents webhook. + EtcdComponents *etcdcomponents.Config } // InitFromFlags initializes the webhook config from the provided CLI flag set. func (cfg *Config) InitFromFlags(fs *flag.FlagSet) { - cfg.Sentinel = &sentinel.Config{} - sentinel.InitFromFlags(fs, cfg.Sentinel) + cfg.EtcdComponents = &etcdcomponents.Config{} + etcdcomponents.InitFromFlags(fs, cfg.EtcdComponents) } // AtLeaseOneEnabled returns true if at least one webhook is enabled. // NOTE for contributors: For every new webhook, add a disjunction condition with the webhook's Enabled field. func (cfg *Config) AtLeaseOneEnabled() bool { - return cfg.Sentinel.Enabled + return cfg.EtcdComponents.Enabled } diff --git a/internal/webhook/sentinel/config.go b/internal/webhook/etcdcomponents/config.go similarity index 63% rename from internal/webhook/sentinel/config.go rename to internal/webhook/etcdcomponents/config.go index 8cdee0ba3..9a31ceaf9 100644 --- a/internal/webhook/sentinel/config.go +++ b/internal/webhook/etcdcomponents/config.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package sentinel +package etcdcomponents import ( "encoding/base64" @@ -16,44 +16,36 @@ import ( ) const ( - enableSentinelWebhookFlagName = "enable-sentinel-webhook" - reconcilerServiceAccountFlagName = "reconciler-service-account" - exemptServiceAccountsFlagName = "sentinel-exempt-service-accounts" - - defaultEnableSentinelWebhook = false - defaultReconcilerServiceAccount = "system:serviceaccount:default:etcd-druid" - - reconcilerServiceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" -) - -var ( - defaultExemptServiceAccounts []string + enableEtcdComponentsWebhookFlagName = "enable-etcd-components-webhook" + reconcilerServiceAccountFlagName = "reconciler-service-account" + etcdComponentsWebhookExemptServiceAccountsFlagName = "etcd-components-webhook-exempt-service-accounts" + defaultEnableWebhook = false + defaultReconcilerServiceAccount = "system:serviceaccount:default:etcd-druid" + reconcilerServiceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) -// Config defines the configuration for the Sentinel Webhook. +// Config defines the configuration for the EtcdComponents Webhook. type Config struct { - // Enabled indicates whether the Sentinel Webhook is enabled. + // Enabled indicates whether the Etcd Components Webhook is enabled. Enabled bool // ReconcilerServiceAccount is the name of the service account used by etcd-druid for reconciling etcd resources. ReconcilerServiceAccount string - // ExemptServiceAccounts is a list of service accounts that are exempt from Sentinel Webhook checks. + // ExemptServiceAccounts is a list of service accounts that are exempt from Etcd Components Webhook checks. ExemptServiceAccounts []string } // InitFromFlags initializes the config from the provided CLI flag set. func InitFromFlags(fs *flag.FlagSet, cfg *Config) { - fs.BoolVar(&cfg.Enabled, enableSentinelWebhookFlagName, defaultEnableSentinelWebhook, - "Enable Sentinel Webhook to prevent unintended changes to resources managed by etcd-druid.") - + fs.BoolVar(&cfg.Enabled, enableEtcdComponentsWebhookFlagName, defaultEnableWebhook, + "Enable Etcd-Components-Webhook to prevent unintended changes to resources managed by etcd-druid.") reconcilerServiceAccount, err := getReconcilerServiceAccountName() if err != nil { reconcilerServiceAccount = defaultReconcilerServiceAccount } fs.StringVar(&cfg.ReconcilerServiceAccount, reconcilerServiceAccountFlagName, reconcilerServiceAccount, fmt.Sprintf("The fully qualified name of the service account used by etcd-druid for reconciling etcd resources. Default: %s", defaultReconcilerServiceAccount)) - - fs.StringSliceVar(&cfg.ExemptServiceAccounts, exemptServiceAccountsFlagName, defaultExemptServiceAccounts, - "The comma-separated list of fully qualified names of service accounts that are exempt from Sentinel Webhook checks.") + fs.StringSliceVar(&cfg.ExemptServiceAccounts, etcdComponentsWebhookExemptServiceAccountsFlagName, []string{}, + "The comma-separated list of fully qualified names of service accounts that are exempt from Etcd-Components-Webhook checks.") } func getReconcilerServiceAccountName() (string, error) { diff --git a/internal/webhook/sentinel/handler.go b/internal/webhook/etcdcomponents/handler.go similarity index 91% rename from internal/webhook/sentinel/handler.go rename to internal/webhook/etcdcomponents/handler.go index 8e62f24ad..30c8fb7e1 100644 --- a/internal/webhook/sentinel/handler.go +++ b/internal/webhook/etcdcomponents/handler.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package sentinel +package etcdcomponents import ( "context" @@ -26,7 +26,9 @@ import ( var allowedOperations = []admissionv1.Operation{admissionv1.Create, admissionv1.Connect} -// Handler is the Sentinel Webhook admission handler. +// Handler is the Etcd Components protection Webhook admission handler. +// All resources that are provisioned by druid as part of etcd cluster provisioning are protected from +// unintended modification or deletion by this admission handler. type Handler struct { client client.Client config *Config @@ -34,7 +36,7 @@ type Handler struct { logger logr.Logger } -// NewHandler creates a new handler for Sentinel Webhook. +// NewHandler creates a new handler for Etcd Components Webhook. func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { return &Handler{ client: mgr.GetClient(), @@ -44,19 +46,11 @@ func NewHandler(mgr manager.Manager, config *Config) (*Handler, error) { }, nil } -/* - handler.Handle -> - 1. check if operation is handled - 2. get object or partial object metadata - 3. pre-checks on object before handling operation - 4. handle operation -*/ - // Handle handles admission requests and prevents unintended changes to resources created by etcd-druid. func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.Response { requestGKString := util.GetGroupKindAsStringFromRequest(req) log := h.logger.WithValues("name", req.Name, "namespace", req.Namespace, "resourceGroupKind", requestGKString, "operation", req.Operation, "user", req.UserInfo.Username) - log.V(1).Info("Sentinel webhook invoked") + log.V(1).Info("EtcdComponents webhook invoked") if ok, response := h.skipValidationForOperations(req.Operation); ok { return *response @@ -67,7 +61,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(util.DetermineStatusCode(err), err) } if partialObjMeta == nil { - return admission.Allowed(fmt.Sprintf("resource: %v is not supported by Sentinel webhook", requestGKString)) + return admission.Allowed(fmt.Sprintf("resource: %v is not supported by EtcdComponents webhook", requestGKString)) } if !isObjManagedByDruid(partialObjMeta.ObjectMeta) { @@ -84,7 +78,7 @@ func (h *Handler) Handle(ctx context.Context, req admission.Request) admission.R // allow changes to resources if Etcd has annotation druid.gardener.cloud/disable-resource-protection is set. if !druidv1alpha1.AreManagedResourcesProtected(etcd.ObjectMeta) { - return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableResourceProtectionAnnotation)) + return admission.Allowed(fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", etcd.Name, druidv1alpha1.DisableEtcdComponentProtectionAnnotation)) } // allow deletion operation on resources if the Etcd is currently being deleted, but only by etcd-druid and exempt service accounts. @@ -117,7 +111,7 @@ func (h *Handler) handleUpdate(req admission.Request, etcd *druidv1alpha1.Etcd) // allow exempt service accounts to make changes to resources, but only if the Etcd is not currently being reconciled. for _, sa := range h.config.ExemptServiceAccounts { if req.UserInfo.Username == sa { - return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", etcd.Name, sa)) + return admission.Allowed(fmt.Sprintf("operations on Etcd %s by service account %s is exempt from EtcdComponents Webhook checks", etcd.Name, sa)) } } diff --git a/internal/webhook/sentinel/handler_test.go b/internal/webhook/etcdcomponents/handler_test.go similarity index 97% rename from internal/webhook/sentinel/handler_test.go rename to internal/webhook/etcdcomponents/handler_test.go index 93f809687..b8f247485 100644 --- a/internal/webhook/sentinel/handler_test.go +++ b/internal/webhook/etcdcomponents/handler_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package sentinel +package etcdcomponents import ( "context" @@ -196,7 +196,7 @@ func TestUnexpectedResourceType(t *testing.T) { }) g.Expect(resp.Allowed).To(BeTrue()) - g.Expect(resp.Result.Message).To(Equal("resource: coordination.k8s.io/Unknown is not supported by Sentinel webhook")) + g.Expect(resp.Result.Message).To(Equal("resource: coordination.k8s.io/Unknown is not supported by EtcdComponents webhook")) } func TestMissingManagedByLabel(t *testing.T) { @@ -269,9 +269,9 @@ func TestHandleUpdate(t *testing.T) { { name: "disable resource protection annotation set", objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, + etcdAnnotations: map[string]string{druidv1alpha1.DisableEtcdComponentProtectionAnnotation: ""}, expectedAllowed: true, - expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableEtcdComponentProtectionAnnotation), expectedCode: http.StatusOK, }, { @@ -301,7 +301,7 @@ func TestHandleUpdate(t *testing.T) { reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, - expectedMessage: fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: fmt.Sprintf("operations on Etcd %s by service account %s is exempt from EtcdComponents Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { @@ -510,9 +510,9 @@ func TestHandleDelete(t *testing.T) { { name: "disable resource protection annotation set", objectLabels: map[string]string{druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, druidv1alpha1.LabelPartOfKey: testEtcdName}, - etcdAnnotations: map[string]string{druidv1alpha1.DisableResourceProtectionAnnotation: ""}, + etcdAnnotations: map[string]string{druidv1alpha1.DisableEtcdComponentProtectionAnnotation: ""}, expectedAllowed: true, - expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableResourceProtectionAnnotation), + expectedMessage: fmt.Sprintf("changes allowed, since Etcd %s has annotation %s", testEtcdName, druidv1alpha1.DisableEtcdComponentProtectionAnnotation), expectedCode: http.StatusOK, }, { @@ -555,7 +555,7 @@ func TestHandleDelete(t *testing.T) { reconcilerServiceAccount: reconcilerServiceAccount, exemptServiceAccounts: exemptServiceAccounts, expectedAllowed: true, - expectedMessage: fmt.Sprintf("operations on Etcd %s by service account %s is exempt from Sentinel Webhook checks", testEtcdName, exemptServiceAccounts[0]), + expectedMessage: fmt.Sprintf("operations on Etcd %s by service account %s is exempt from EtcdComponents Webhook checks", testEtcdName, exemptServiceAccounts[0]), expectedCode: http.StatusOK, }, { diff --git a/internal/webhook/sentinel/register.go b/internal/webhook/etcdcomponents/register.go similarity index 86% rename from internal/webhook/sentinel/register.go rename to internal/webhook/etcdcomponents/register.go index 01bb85e12..ad2acd27b 100644 --- a/internal/webhook/sentinel/register.go +++ b/internal/webhook/etcdcomponents/register.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package sentinel +package etcdcomponents import ( "sigs.k8s.io/controller-runtime/pkg/manager" @@ -11,9 +11,9 @@ import ( const ( // handlerName is the name of the webhook handler. - handlerName = "sentinel-webhook" + handlerName = "etcd-components-webhook" // WebhookPath is the path at which the handler should be registered. - webhookPath = "/webhooks/sentinel" + webhookPath = "/webhooks/etcdcomponents" ) // RegisterWithManager registers Handler to the given manager. diff --git a/internal/webhook/register.go b/internal/webhook/register.go index 81d9f39c8..395c0a116 100644 --- a/internal/webhook/register.go +++ b/internal/webhook/register.go @@ -5,7 +5,7 @@ package webhook import ( - "github.com/gardener/etcd-druid/internal/webhook/sentinel" + "github.com/gardener/etcd-druid/internal/webhook/etcdcomponents" "golang.org/x/exp/slog" ctrl "sigs.k8s.io/controller-runtime" @@ -13,17 +13,17 @@ import ( // Register registers all etcd-druid webhooks with the controller manager. func Register(mgr ctrl.Manager, config *Config) error { - // Add sentinel webhook to the manager - if config.Sentinel.Enabled { - sentinelWebhook, err := sentinel.NewHandler( + // Add Etcd Components webhook to the manager + if config.EtcdComponents.Enabled { + etcdComponentsWebhook, err := etcdcomponents.NewHandler( mgr, - config.Sentinel, + config.EtcdComponents, ) if err != nil { return err } - slog.Info("Registering Sentinel Webhook with manager") - return sentinelWebhook.RegisterWithManager(mgr) + slog.Info("Registering EtcdComponents Webhook with manager") + return etcdComponentsWebhook.RegisterWithManager(mgr) } return nil } diff --git a/internal/webhook/util/decode.go b/internal/webhook/util/decode.go index a84c260d1..d7abe4030 100644 --- a/internal/webhook/util/decode.go +++ b/internal/webhook/util/decode.go @@ -7,7 +7,6 @@ import ( druiderr "github.com/gardener/etcd-druid/internal/errors" admissionv1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" @@ -62,7 +61,6 @@ func (d *RequestDecoder) DecodeRequestObjectAsPartialObjectMetadata(ctx context. rbacv1.SchemeGroupVersion.WithKind("RoleBinding").GroupKind(), appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), policyv1.SchemeGroupVersion.WithKind("PodDisruptionBudget").GroupKind(), - batchv1.SchemeGroupVersion.WithKind("Job").GroupKind(), coordinationv1.SchemeGroupVersion.WithKind("Lease").GroupKind(): return d.doDecodeRequestObjAsPartialObjectMeta(req) case autoscalingv1.SchemeGroupVersion.WithKind("Scale").GroupKind(): diff --git a/skaffold.yaml b/skaffold.yaml index ea8721e91..7d7761821 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -39,14 +39,14 @@ profiles: value: etcd: etcdStatusSyncPeriod: 5s - - name: enable-sentinel-webhook + - name: enable-etcdcomponents-webhook activation: - - env: "DRUID_ENABLE_SENTINEL_WEBHOOK=true" + - env: "DRUID_ENABLE_ETCD_COMPONENTS_WEBHOOK=true" patches: - op: add path: /deploy/helm/releases/0/setValues/webhooks value: - sentinel: + etcdcomponents: enabled: true - name: do-not-use-feature-gates activation: From 553736a062a53638e6ffbd9976a0b22d3f46cf32 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 4 Jun 2024 18:59:15 +0530 Subject: [PATCH 227/235] Add log line for statefulset component PreSync --- internal/component/statefulset/statefulset.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index 48eeb9223..dc89e4421 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -76,6 +76,8 @@ func (r _resource) GetExistingResourceNames(ctx component.OperatorContext, etcdO // is different from the label selector required to be applied on it. This is because the statefulset's // spec.selector field is immutable and cannot be updated on the existing statefulset. func (r _resource) PreSync(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd) error { + ctx.Logger.Info("Running pre-sync for StatefulSet", "name", druidv1alpha1.GetStatefulSetName(etcd.ObjectMeta), "namespace", druidv1alpha1.GetNamespaceName(etcd.ObjectMeta)) + sts, err := r.getExistingStatefulSet(ctx, etcd.ObjectMeta) if err != nil { return druiderr.WrapError(err, From 4c6cf00e357c464c04560c1506b6b43969e68151 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 11 Jun 2024 13:49:00 +0530 Subject: [PATCH 228/235] Statefulset PreSync now checks sts status instead of pod metadata, to determine pod updation --- internal/component/statefulset/statefulset.go | 22 ++++++++----------- test/e2e/etcd_backup_test.go | 3 --- test/e2e/etcd_compaction_test.go | 3 --- test/e2e/suite_test.go | 2 +- test/e2e/utils.go | 5 ----- test/it/controller/etcd/helper.go | 4 ++++ 6 files changed, 14 insertions(+), 25 deletions(-) diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index dc89e4421..78dbe064b 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -17,7 +17,6 @@ import ( "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/imagevector" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -278,23 +277,20 @@ func (r _resource) checkAndPatchStsPodLabelsOnMismatch(ctx component.OperatorCon } func (r _resource) areStatefulSetPodsUpdatedAndReady(ctx component.OperatorContext, sts *appsv1.StatefulSet) (bool, error) { - if sts.Status.UpdatedReplicas != *sts.Spec.Replicas { + if err := r.client.Get(ctx, getObjectKey(sts.ObjectMeta), sts); err != nil { + return false, err + } + if sts.Status.ObservedGeneration < sts.Generation { return false, nil } - podList := &corev1.PodList{} - if err := r.client.List(ctx, podList, client.InNamespace(sts.Namespace), client.MatchingLabels(sts.Spec.Template.Labels)); err != nil { - return false, err + if sts.Status.UpdateRevision != sts.Status.CurrentRevision { + return false, nil } - if len(podList.Items) != int(*sts.Spec.Replicas) { + if sts.Status.UpdatedReplicas != *sts.Spec.Replicas { return false, nil } - for _, pod := range podList.Items { - if !utils.ContainsLabel(pod.Labels, appsv1.StatefulSetRevisionLabel, sts.Status.UpdateRevision) { - return false, nil - } - if !utils.HasPodReadyConditionTrue(&pod) { - return false, nil - } + if sts.Status.ReadyReplicas < *sts.Spec.Replicas { + return false, nil } return true, nil } diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index 9fb89f491..4d00e0099 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -50,9 +50,6 @@ var _ = Describe("Etcd Backup", func() { provider := p Context(fmt.Sprintf("with provider %s", provider.Name), func() { BeforeEach(func() { - cl, err = getKubernetesClient(kubeconfigPath) - Expect(err).ShouldNot(HaveOccurred()) - etcdName = fmt.Sprintf("etcd-%s", provider.Name) storageContainer = getEnvAndExpectNoError(envStorageContainer) diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index 9d34a8217..758f555e6 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -43,9 +43,6 @@ var _ = Describe("Etcd Compaction", func() { provider := p Context(fmt.Sprintf("with provider %s", provider.Name), func() { BeforeEach(func() { - cl, err = getKubernetesClient(kubeconfigPath) - Expect(err).ShouldNot(HaveOccurred()) - etcdName = fmt.Sprintf("etcd-%s", provider.Name) storageContainer = getEnvAndExpectNoError(envStorageContainer) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index d4bf5898b..e34dd9900 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -53,7 +53,6 @@ var ( etcdValuePrefix = "bar" providers []TestProvider - err error ) func TestIntegration(t *testing.T) { @@ -62,6 +61,7 @@ func TestIntegration(t *testing.T) { } var _ = BeforeSuite(func() { + var err error ctx := context.Background() providers, err = getProviders() diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 464928a97..fe163d5ac 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -20,7 +20,6 @@ import ( "github.com/gardener/etcd-backup-restore/pkg/snapstore" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/go-logr/logr" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -86,9 +85,6 @@ var ( "garden.sapcloud.io/role": "controlplane", roleLabelKey: defaultRoleLabelValue, } - annotations = map[string]string{ - v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, - } stsLabels = map[string]string{ "app": "etcd-statefulset", @@ -166,7 +162,6 @@ func getEmptyEtcd(name, namespace string) *v1alpha1.Etcd { func getDefaultEtcd(name, namespace, container, prefix string, provider TestProvider) *v1alpha1.Etcd { etcd := getEmptyEtcd(name, namespace) - etcd.Annotations = annotations etcd.Spec.Annotations = stsAnnotations labelsCopy := make(map[string]string) diff --git a/test/it/controller/etcd/helper.go b/test/it/controller/etcd/helper.go index e5effa4de..014e14d09 100644 --- a/test/it/controller/etcd/helper.go +++ b/test/it/controller/etcd/helper.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + package etcd import ( From 5cc2aa76d23534bae6d1fa187ddedbbdc6ba1954 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 11 Jun 2024 22:34:32 +0530 Subject: [PATCH 229/235] Fix it tests by simulating sts status changes upon spec changes --- test/it/controller/etcd/helper.go | 11 +++++++---- test/it/controller/etcd/reconciler_test.go | 6 ++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/it/controller/etcd/helper.go b/test/it/controller/etcd/helper.go index 014e14d09..bb33ba1e8 100644 --- a/test/it/controller/etcd/helper.go +++ b/test/it/controller/etcd/helper.go @@ -106,10 +106,13 @@ func updateAndGetStsRevision(ctx context.Context, t *testing.T, cl client.Client sts := &appsv1.StatefulSet{} g.Expect(cl.Get(ctx, client.ObjectKey{Name: druidv1alpha1.GetStatefulSetName(etcdInstance.ObjectMeta), Namespace: etcdInstance.Namespace}, sts)).To(Succeed()) originalSts := sts.DeepCopy() + sts.Status.ObservedGeneration = sts.Generation sts.Status.Replicas = etcdInstance.Spec.Replicas sts.Status.ReadyReplicas = etcdInstance.Spec.Replicas sts.Status.UpdatedReplicas = etcdInstance.Spec.Replicas - sts.Status.UpdateRevision = fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 4)) + revision := fmt.Sprintf("%s-%s", sts.Name, generateRandomAlphanumericString(t, 4)) + sts.Status.CurrentRevision = revision + sts.Status.UpdateRevision = revision g.Expect(cl.Status().Patch(ctx, sts, client.MergeFrom(originalSts))).To(Succeed()) return sts.Status.UpdateRevision } @@ -122,9 +125,9 @@ func createStsPods(ctx context.Context, t *testing.T, cl client.Client, etcdObje for i := 0; i < int(stsReplicas); i++ { podName := fmt.Sprintf("%s-%d", sts.Name, i) podLabels := utils.MergeMaps(sts.Spec.Template.Labels, etcdObjectMeta.Labels, map[string]string{ - "apps.kubernetes.io/pod-index": strconv.Itoa(i), - "statefulset.kubernetes.io/pod-name": podName, - "controller-revision-hash": stsUpdateRevision, + appsv1.PodIndexLabel: strconv.Itoa(i), + appsv1.StatefulSetPodNameLabel: podName, + appsv1.StatefulSetRevisionLabel: stsUpdateRevision, }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 230618d98..57874afa9 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -210,11 +210,12 @@ func testEtcdSpecUpdateWhenNoReconcileOperationAnnotationIsSet(t *testing.T, tes // update etcdInstance spec without reconcile operation annotation set originalEtcdInstance := etcdInstance.DeepCopy() metricsLevelExtensive := druidv1alpha1.Extensive - etcdInstance.Spec.Etcd = druidv1alpha1.EtcdConfig{Metrics: &metricsLevelExtensive} + etcdInstance.Spec.Etcd.Metrics = &metricsLevelExtensive g.Expect(cl.Patch(ctx, etcdInstance, client.MergeFrom(originalEtcdInstance))).To(Succeed()) // ***************** test etcd spec reconciliation ***************** assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 2*time.Second, 2*time.Second) + _ = updateAndGetStsRevision(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), etcdInstance) assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(1), 5*time.Second, 1*time.Second) // ensure that sts generation does not change, ie, it should remain 1, as sts is not updated after etcd spec change without reconcile operation annotation assertStatefulSetGeneration(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), 1, 30*time.Second, 2*time.Second) @@ -238,7 +239,7 @@ func testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet(t *testing.T, testN // update etcdInstance spec with reconcile operation annotation also set originalEtcdInstance := etcdInstance.DeepCopy() metricsLevelExtensive := druidv1alpha1.Extensive - etcdInstance.Spec.Etcd = druidv1alpha1.EtcdConfig{Metrics: &metricsLevelExtensive} + etcdInstance.Spec.Etcd.Metrics = &metricsLevelExtensive etcdInstance.Annotations = map[string]string{ v1beta1constants.GardenerOperation: v1beta1constants.GardenerOperationReconcile, } @@ -246,6 +247,7 @@ func testEtcdSpecUpdateWhenReconcileOperationAnnotationIsSet(t *testing.T, testN // ***************** test etcd spec reconciliation ***************** assertAllComponentsExists(ctx, t, reconcilerTestEnv, etcdInstance, 30*time.Minute, 2*time.Second) + _ = updateAndGetStsRevision(ctx, t, reconcilerTestEnv.itTestEnv.GetClient(), etcdInstance) assertETCDObservedGeneration(t, reconcilerTestEnv.itTestEnv.GetClient(), client.ObjectKeyFromObject(etcdInstance), pointer.Int64(2), 2*time.Minute, 1*time.Second) expectedLastOperation := &druidv1alpha1.LastOperation{ Type: druidv1alpha1.LastOperationTypeReconcile, From 4aa2d362c7b610bb574ae0d8a3df4fa99f5661a5 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Tue, 11 Jun 2024 22:40:15 +0530 Subject: [PATCH 230/235] Fix e2e tests --- test/e2e/etcd_backup_test.go | 1 - test/e2e/etcd_compaction_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index 4d00e0099..4beab1a58 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -41,7 +41,6 @@ var _ = Describe("Etcd Backup", func() { Expect(err).ToNot(HaveOccurred()) var ( - cl client.Client etcdName string storageContainer string ) diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index 758f555e6..7d41b2dde 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -34,7 +34,6 @@ var _ = Describe("Etcd Compaction", func() { Expect(err).ToNot(HaveOccurred()) var ( - cl client.Client etcdName string storageContainer string ) From 63cfadded13261b0586d21d57f65851992c16e2c Mon Sep 17 00:00:00 2001 From: Marcel Boehm Date: Wed, 12 Jun 2024 11:52:55 +0530 Subject: [PATCH 231/235] Map `stackit` infra provider to S3 --- internal/store/store.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/store/store.go b/internal/store/store.go index 7eb282dc1..8c31f0b09 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -33,6 +33,7 @@ const ( openstack = "openstack" dell = "dell" openshift = "openshift" + stackit = "stackit" ) const ( @@ -63,6 +64,9 @@ func StorageProviderFromInfraProvider(infra *druidv1alpha1.StorageProvider) (str switch *infra { case aws, S3: return S3, nil + // S3-compatible providers + case stackit: + return S3, nil case azure, ABS: return ABS, nil case alicloud, OSS: From aa41f39c8341f45955a768474cd41f4f5c14771c Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Fri, 14 Jun 2024 15:46:23 +0530 Subject: [PATCH 232/235] Fix status member checks --- Makefile | 4 ++-- api/v1alpha1/helper.go | 4 ++-- internal/component/memberlease/memberlease.go | 2 +- .../component/memberlease/memberlease_test.go | 4 ++-- internal/component/statefulset/statefulset.go | 20 ++++++++++++---- internal/controller/etcd/reconcile_spec.go | 11 +++++---- internal/controller/etcd/reconciler.go | 2 +- internal/controller/utils/reconciler.go | 4 ++++ internal/health/etcdmember/check_ready.go | 24 ++++++++++++------- test/it/controller/etcd/reconciler_test.go | 2 +- 10 files changed, 49 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 1e368d525..f12ff9f27 100644 --- a/Makefile +++ b/Makefile @@ -161,11 +161,11 @@ deploy: $(SKAFFOLD) $(HELM) .PHONY: deploy-dev deploy-dev: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) dev -m etcd-druid --trigger='manual' + $(SKAFFOLD) dev --cleanup=false -m etcd-druid --trigger='manual' .PHONY: deploy-debug deploy-debug: $(SKAFFOLD) $(HELM) - $(SKAFFOLD) debug -m etcd-druid + $(SKAFFOLD) debug --cleanup=false -m etcd-druid .PHONY: undeploy undeploy: $(SKAFFOLD) $(HELM) diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index e9468b44a..27eb69efa 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -45,9 +45,9 @@ func GetOrdinalPodName(etcdObjMeta metav1.ObjectMeta, ordinal int) string { } // GetMemberLeaseNames returns the name of member leases for the Etcd. -func GetMemberLeaseNames(etcdObjMeta metav1.ObjectMeta, replicas int) []string { +func GetMemberLeaseNames(etcdObjMeta metav1.ObjectMeta, replicas int32) []string { leaseNames := make([]string, 0, replicas) - for i := 0; i < replicas; i++ { + for i := 0; i < int(replicas); i++ { leaseNames = append(leaseNames, fmt.Sprintf("%s-%d", etcdObjMeta.Name, i)) } return leaseNames diff --git a/internal/component/memberlease/memberlease.go b/internal/component/memberlease/memberlease.go index 3fb49275a..68069548b 100644 --- a/internal/component/memberlease/memberlease.go +++ b/internal/component/memberlease/memberlease.go @@ -127,7 +127,7 @@ func buildResource(etcd *druidv1alpha1.Etcd, lease *coordinationv1.Lease) { } func getObjectKeys(etcd *druidv1alpha1.Etcd) []client.ObjectKey { - leaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas)) + leaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, etcd.Spec.Replicas) objectKeys := make([]client.ObjectKey, 0, len(leaseNames)) for _, leaseName := range leaseNames { objectKeys = append(objectKeys, client.ObjectKey{Name: leaseName, Namespace: etcd.Namespace}) diff --git a/internal/component/memberlease/memberlease_test.go b/internal/component/memberlease/memberlease_test.go index 43a1a2468..2246b0f76 100644 --- a/internal/component/memberlease/memberlease_test.go +++ b/internal/component/memberlease/memberlease_test.go @@ -95,7 +95,7 @@ func TestGetExistingResourceNames(t *testing.T) { testutils.CheckDruidError(g, tc.expectedErr, err) } else { g.Expect(err).To(BeNil()) - expectedLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas))[:tc.numExistingLeases] + expectedLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, etcd.Spec.Replicas)[:tc.numExistingLeases] g.Expect(memberLeaseNames).To(Equal(expectedLeaseNames)) } }) @@ -309,7 +309,7 @@ func newMemberLeases(etcd *druidv1alpha1.Etcd, numLeases int) ([]*coordinationv1 if numLeases > int(etcd.Spec.Replicas) { return nil, errors.New("number of requested leases is greater than the etcd replicas") } - memberLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas)) + memberLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, etcd.Spec.Replicas) leases := make([]*coordinationv1.Lease, 0, numLeases) for i := 0; i < numLeases; i++ { lease := &coordinationv1.Lease{ diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index 78dbe064b..f5f661e5c 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -98,7 +98,7 @@ func (r _resource) PreSync(ctx component.OperatorContext, etcd *druidv1alpha1.Et } // check if pods have been updated with new labels and become ready. - podsUpdatedAndReady, err := r.areStatefulSetPodsUpdatedAndReady(ctx, sts) + podsUpdatedAndReady, err := r.areStatefulSetPodsUpdatedAndReady(ctx, etcd, sts) if err != nil { return druiderr.WrapError(err, ErrPreSyncStatefulSet, @@ -112,6 +112,8 @@ func (r _resource) PreSync(ctx component.OperatorContext, etcd *druidv1alpha1.Et "PreSync", errMessage, ) + } else { + ctx.Logger.Info("StatefulSet pods are updated with new pod labels and ready", "objectKey", getObjectKey(etcd.ObjectMeta)) } // if sts label selector needs to be changed, then delete the statefulset, but keeping the pods intact. @@ -264,11 +266,11 @@ func isPeerTLSEnablementPending(peerTLSEnabledStatusFromMembers bool, etcd *drui } func (r _resource) checkAndPatchStsPodLabelsOnMismatch(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) error { - desiredPodLabels := utils.MergeMaps(etcd.Spec.Labels, getStatefulSetLabels(etcd.Name)) - if !utils.ContainsAllDesiredLabels(sts.Spec.Template.Labels, desiredPodLabels) { + desiredPodTemplateLabels := getDesiredPodTemplateLabels(etcd) + if !utils.ContainsAllDesiredLabels(sts.Spec.Template.Labels, desiredPodTemplateLabels) { ctx.Logger.Info("Patching StatefulSet with new pod labels", "objectKey", getObjectKey(etcd.ObjectMeta)) originalSts := sts.DeepCopy() - sts.Spec.Template.Labels = utils.MergeMaps(sts.Spec.Template.Labels, desiredPodLabels) + sts.Spec.Template.Labels = utils.MergeMaps(sts.Spec.Template.Labels, desiredPodTemplateLabels) if err := r.client.Patch(ctx, sts, client.MergeFrom(originalSts)); err != nil { return err } @@ -276,10 +278,18 @@ func (r _resource) checkAndPatchStsPodLabelsOnMismatch(ctx component.OperatorCon return nil } -func (r _resource) areStatefulSetPodsUpdatedAndReady(ctx component.OperatorContext, sts *appsv1.StatefulSet) (bool, error) { +func getDesiredPodTemplateLabels(etcd *druidv1alpha1.Etcd) map[string]string { + return utils.MergeMaps(etcd.Spec.Labels, getStatefulSetLabels(etcd.Name)) +} + +func (r _resource) areStatefulSetPodsUpdatedAndReady(ctx component.OperatorContext, etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) (bool, error) { if err := r.client.Get(ctx, getObjectKey(sts.ObjectMeta), sts); err != nil { return false, err } + desiredPodTemplateLabels := getDesiredPodTemplateLabels(etcd) + if !utils.ContainsAllDesiredLabels(sts.Spec.Template.Labels, desiredPodTemplateLabels) { + return false, nil + } if sts.Status.ObservedGeneration < sts.Generation { return false, nil } diff --git a/internal/controller/etcd/reconcile_spec.go b/internal/controller/etcd/reconcile_spec.go index e828a4f4d..80e99c3d1 100644 --- a/internal/controller/etcd/reconcile_spec.go +++ b/internal/controller/etcd/reconcile_spec.go @@ -22,7 +22,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -const retryInterval = 1 * time.Second +// syncRetryInterval will be used by both sync and preSync stages for a component and should be used when there is a need to requeue for retrying after a specific interval. +const syncRetryInterval = 10 * time.Second func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult { reconcileStepFns := []reconcileFn{ @@ -92,8 +93,8 @@ func (r *Reconciler) preSyncEtcdResources(ctx component.OperatorContext, etcdObj op := r.operatorRegistry.GetOperator(kind) if err := op.PreSync(ctx, etcd); err != nil { if druiderr.IsRequeueAfterError(err) { - ctx.Logger.Info("retrying pre-sync of component", "kind", kind, "retryInterval", retryInterval.String()) - return ctrlutils.ReconcileAfter(retryInterval, fmt.Sprintf("retrying pre-sync of component %s after %s", kind, retryInterval.String())) + ctx.Logger.Info("retrying pre-sync of component", "kind", kind, "syncRetryInterval", syncRetryInterval.String()) + return ctrlutils.ReconcileAfter(syncRetryInterval, fmt.Sprintf("requeueing pre-sync of component %s to be retried after %s", kind, syncRetryInterval.String())) } ctx.Logger.Error(err, "failed to sync etcd resource", "kind", kind) return ctrlutils.ReconcileWithError(err) @@ -112,8 +113,8 @@ func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey op := r.operatorRegistry.GetOperator(kind) if err := op.Sync(ctx, etcd); err != nil { if druiderr.IsRequeueAfterError(err) { - ctx.Logger.Info("retrying sync of component", "kind", kind, "retryInterval", retryInterval.String()) - return ctrlutils.ReconcileAfter(retryInterval, fmt.Sprintf("retrying sync of component %s after %s", kind, retryInterval.String())) + ctx.Logger.Info("retrying sync of component", "kind", kind, "syncRetryInterval", syncRetryInterval.String()) + return ctrlutils.ReconcileAfter(syncRetryInterval, fmt.Sprintf("retrying sync of component %s after %s", kind, syncRetryInterval.String())) } ctx.Logger.Error(err, "failed to sync etcd resource", "kind", kind) return ctrlutils.ReconcileWithError(err) diff --git a/internal/controller/etcd/reconciler.go b/internal/controller/etcd/reconciler.go index 1e640e55f..e09b7396e 100644 --- a/internal/controller/etcd/reconciler.go +++ b/internal/controller/etcd/reconciler.go @@ -103,7 +103,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.logger.Error(result.GetCombinedError(), "Failed to reconcile status") return result.ReconcileResult() } - if reconcileSpecResult.HasErrors() { + if reconcileSpecResult.NeedsRequeue() { return reconcileSpecResult.ReconcileResult() } return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult() diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 95907186e..22ec754a6 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -79,6 +79,10 @@ func (r ReconcileStepResult) HasErrors() bool { return len(r.errs) > 0 } +func (r ReconcileStepResult) NeedsRequeue() bool { + return r.HasErrors() || r.result.Requeue || r.result.RequeueAfter > 0 +} + // GetDescription returns the description of the reconcile step. func (r ReconcileStepResult) GetDescription() string { if len(r.errs) > 0 { diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index 3ab0b9541..dfd225daa 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -6,11 +6,11 @@ package etcdmember import ( "context" + "fmt" "strings" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - "github.com/gardener/etcd-druid/internal/common" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" @@ -35,16 +35,22 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul checkTime = TimeNow().UTC() ) - leases := &coordinationv1.LeaseList{} - if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ - druidv1alpha1.LabelComponentKey: common.ComponentNameMemberLease, - druidv1alpha1.LabelManagedByKey: druidv1alpha1.LabelManagedByValue, - druidv1alpha1.LabelPartOfKey: etcd.Name, - }); err != nil { - r.logger.Error(err, "failed to get leases for etcd member readiness check") + leaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, etcd.Spec.Replicas) + leases := make([]*coordinationv1.Lease, 0, len(leaseNames)) + for _, leaseName := range leaseNames { + lease := &coordinationv1.Lease{} + if err := r.cl.Get(ctx, kutil.Key(etcd.Namespace, leaseName), lease); err != nil { + if apierrors.IsNotFound(err) { + r.logger.Error(fmt.Errorf("lease not found"), "name", leaseName) + continue + } + r.logger.Error(err, "failed to get lease", "name", leaseName) + continue + } + leases = append(leases, lease) } - for _, lease := range leases.Items { + for _, lease := range leases { var ( id, role = separateIdFromRole(lease.Spec.HolderIdentity) res = &result{ diff --git a/test/it/controller/etcd/reconciler_test.go b/test/it/controller/etcd/reconciler_test.go index 57874afa9..e5698f8a8 100644 --- a/test/it/controller/etcd/reconciler_test.go +++ b/test/it/controller/etcd/reconciler_test.go @@ -418,7 +418,7 @@ func TestEtcdStatusReconciliation(t *testing.T) { } func testConditionsAndMembersWhenAllMemberLeasesAreActive(t *testing.T, etcd *druidv1alpha1.Etcd, reconcilerTestEnv ReconcilerTestEnv) { - memberLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, int(etcd.Spec.Replicas)) + memberLeaseNames := druidv1alpha1.GetMemberLeaseNames(etcd.ObjectMeta, etcd.Spec.Replicas) testNs := etcd.Namespace clock := testclock.NewFakeClock(time.Now().Round(time.Second)) mlcs := []etcdMemberLeaseConfig{ From 006e470ac030482815864b813c4897f3ac1ae157 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 17 Jun 2024 19:37:46 +0530 Subject: [PATCH 233/235] addressed review comments, added logchecker to golangci --- .golangci.yaml | 5 + internal/component/statefulset/statefulset.go | 4 +- internal/controller/etcd/register.go | 65 +++++-- internal/controller/etcd/register_test.go | 163 ++++++++++++------ internal/controller/utils/reconciler.go | 1 + internal/health/etcdmember/check_ready.go | 2 +- 6 files changed, 169 insertions(+), 71 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index df3aeef25..9e174cfe5 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -7,6 +7,7 @@ linters: - unused enable: - revive + - loggercheck issues: exclude-use-default: false @@ -42,3 +43,7 @@ linters-settings: - name: context-as-argument - name: early-return - name: exported + loggercheck: + no-printf-like: true + logr: true + zap: true \ No newline at end of file diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index f5f661e5c..7165d8c92 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -296,10 +296,10 @@ func (r _resource) areStatefulSetPodsUpdatedAndReady(ctx component.OperatorConte if sts.Status.UpdateRevision != sts.Status.CurrentRevision { return false, nil } - if sts.Status.UpdatedReplicas != *sts.Spec.Replicas { + if sts.Spec.Replicas != nil && sts.Status.UpdatedReplicas != *sts.Spec.Replicas { return false, nil } - if sts.Status.ReadyReplicas < *sts.Spec.Replicas { + if sts.Spec.Replicas != nil && sts.Status.ReadyReplicas < *sts.Spec.Replicas { return false, nil } return true, nil diff --git a/internal/controller/etcd/register.go b/internal/controller/etcd/register.go index 488438568..d1103c92e 100644 --- a/internal/controller/etcd/register.go +++ b/internal/controller/etcd/register.go @@ -40,27 +40,36 @@ func (r *Reconciler) RegisterWithManager(mgr ctrl.Manager) error { // 2. generic events are never reconciled. If there is a need in future to react to generic events then this should be changed. // Conditions for reconciliation: // Scenario 1: {Auto-Reconcile: false, Reconcile-Annotation-Present: false, Spec-Updated: true/false, Status-Updated: true/false, update-event-reconciled: false} -// Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true/false, Status-Updated: true/false, update-event-reconciled: true} -// Scenario 3: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: NA}, This condition cannot happen. In case of a controller restart there will only be a CreateEvent. -// Scenario 4: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: false, update-event-reconciled: true} -// Scenario 5: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: true, update-event-reconciled: false} -// Scenario 6: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: true, update-event-reconciled: true} -// Scenario 7: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: false} -// Scenario 8: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true/false, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 2: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: true, Last-Reconcile-Succeeded: true/false, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 3: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Last-Reconcile-Succeeded: true, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 4: {Auto-Reconcile: false, Reconcile-Annotation-Present: true, Spec-Updated: false, Last-Reconcile-Succeeded: false, Status-Updated: true/false, update-event-reconciled: false} +// Scenario 5: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: false, update-event-reconciled: NA}, This condition cannot happen. In case of a controller restart there will only be a CreateEvent. +// Scenario 6: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: true, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 7: {Auto-Reconcile: true, Reconcile-Annotation-Present: false, Spec-Updated: false, Status-Updated: true/false, update-event-reconciled: false} +// Scenario 8: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: true, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 9: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Last-Reconcile-Succeeded: true, Status-Updated: true/false, update-event-reconciled: true} +// Scenario 9: {Auto-Reconcile: true, Reconcile-Annotation-Present: true, Spec-Updated: false, Last-Reconcile-Succeeded: false, Status-Updated: true/false, update-event-reconciled: false} func (r *Reconciler) buildPredicate() predicate.Predicate { - // If there is a spec change (irrespective of status change) and if there is an update event then it will trigger a reconcile only when either - // auto-reconcile has been enabled or an operator has added the reconcile annotation to the etcd resource. - onSpecChangePredicate := predicate.And( - predicate.Or( - r.hasReconcileAnnotation(), - r.autoReconcileEnabled(), - ), + // If the reconcile annotation is set then only allow reconciliation if one of the conditions is true: + // 1. There has been a spec update. + // 2. The last reconcile operation has finished. + // It is possible that during the previous reconcile one of the steps errored out. This gets captured in etcd.Status.LastOperation. + // Update of status will generate an event. This event should not trigger a reconcile especially when the reconcile annotation has still + // not been removed (since the last reconcile is not yet successfully completed). + onReconcileAnnotationSetPredicate := predicate.And( + r.hasReconcileAnnotation(), + predicate.Or(lastReconcileHasFinished(), specUpdated()), + ) + + // If auto-reconcile has been enabled then it should allow reconciliation only on spec change. + autoReconcileOnSpecChangePredicate := predicate.And( + r.autoReconcileEnabled(), specUpdated(), ) return predicate.Or( - r.hasReconcileAnnotation(), - onSpecChangePredicate, + onReconcileAnnotationSetPredicate, + autoReconcileOnSpecChangePredicate, ) } @@ -110,3 +119,27 @@ func specUpdated() predicate.Predicate { func hasSpecChanged(updateEvent event.UpdateEvent) bool { return updateEvent.ObjectNew.GetGeneration() != updateEvent.ObjectOld.GetGeneration() } + +func lastReconcileHasFinished() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return hasLastReconcileFinished(updateEvent) + }, + CreateFunc: func(createEvent event.CreateEvent) bool { return false }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { return false }, + GenericFunc: func(genericEvent event.GenericEvent) bool { return false }, + } +} + +func hasLastReconcileFinished(updateEvent event.UpdateEvent) bool { + newEtcd, ok := updateEvent.ObjectNew.(*druidv1alpha1.Etcd) + // return false if either the object is not an etcd resource or it has not been reconciled yet. + if !ok || newEtcd.Status.LastOperation == nil { + return false + } + lastOpType := newEtcd.Status.LastOperation.Type + lastOpState := newEtcd.Status.LastOperation.State + + return lastOpType == druidv1alpha1.LastOperationTypeReconcile && + lastOpState == druidv1alpha1.LastOperationStateSucceeded +} diff --git a/internal/controller/etcd/register_test.go b/internal/controller/etcd/register_test.go index dda981183..3e50f7d1b 100644 --- a/internal/controller/etcd/register_test.go +++ b/internal/controller/etcd/register_test.go @@ -9,6 +9,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" mockmanager "github.com/gardener/etcd-druid/internal/mock/controller-runtime/manager" + "github.com/gardener/etcd-druid/internal/utils" testutils "github.com/gardener/etcd-druid/test/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" . "github.com/onsi/gomega" @@ -18,20 +19,42 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" ) +type predicateTestCase struct { + name string + etcdSpecChanged bool + etcdStatusChanged bool + lastOperationState *druidv1alpha1.LastOperationState + // expected behavior for different event types + shouldAllowCreateEvent bool + shouldAllowDeleteEvent bool + shouldAllowGenericEvent bool + shouldAllowUpdateEvent bool +} + func TestBuildPredicateWithOnlyAutoReconcileEnabled(t *testing.T) { - testCases := []struct { - name string - etcdSpecChanged bool - etcdStatusChanged bool - // expected behavior for different event types - shouldAllowCreateEvent bool - shouldAllowDeleteEvent bool - shouldAllowGenericEvent bool - shouldAllowUpdateEvent bool - }{ + testCases := []predicateTestCase{ { - name: "only spec has changed", + name: "only spec has changed and previous reconciliation is in progress", + etcdSpecChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateProcessing), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "only spec has changed and previous reconciliation has completed", + etcdSpecChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateSucceeded), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "only spec has changed and previous reconciliation has errored", etcdSpecChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateError), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, @@ -46,9 +69,10 @@ func TestBuildPredicateWithOnlyAutoReconcileEnabled(t *testing.T) { shouldAllowUpdateEvent: false, }, { - name: "both spec and status have changed", + name: "both spec and status have changed and previous reconciliation is in progress", etcdSpecChanged: true, etcdStatusChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateProcessing), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, @@ -68,7 +92,7 @@ func TestBuildPredicateWithOnlyAutoReconcileEnabled(t *testing.T) { predicate := r.buildPredicate() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, false) + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, tc.lastOperationState, false) g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) @@ -78,16 +102,7 @@ func TestBuildPredicateWithOnlyAutoReconcileEnabled(t *testing.T) { } func TestBuildPredicateWithNoAutoReconcileAndNoReconcileAnnot(t *testing.T) { - testCases := []struct { - name string - etcdSpecChanged bool - etcdStatusChanged bool - // expected behavior for different event types - shouldAllowCreateEvent bool - shouldAllowDeleteEvent bool - shouldAllowGenericEvent bool - shouldAllowUpdateEvent bool - }{ + testCases := []predicateTestCase{ { name: "only spec has changed", etcdSpecChanged: true, @@ -127,7 +142,7 @@ func TestBuildPredicateWithNoAutoReconcileAndNoReconcileAnnot(t *testing.T) { predicate := r.buildPredicate() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, false) + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, tc.lastOperationState, false) g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) @@ -137,27 +152,38 @@ func TestBuildPredicateWithNoAutoReconcileAndNoReconcileAnnot(t *testing.T) { } func TestBuildPredicateWithNoAutoReconcileButReconcileAnnotPresent(t *testing.T) { - testCases := []struct { - name string - etcdSpecChanged bool - etcdStatusChanged bool - // expected behavior for different event types - shouldAllowCreateEvent bool - shouldAllowDeleteEvent bool - shouldAllowGenericEvent bool - shouldAllowUpdateEvent bool - }{ + testCases := []predicateTestCase{ { - name: "only spec has changed", + name: "only spec has changed and previous reconciliation is in progress", etcdSpecChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateProcessing), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, shouldAllowUpdateEvent: true, }, { - name: "only status has changed", + name: "only spec has changed and previous reconciliation is completed", + etcdSpecChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateSucceeded), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: true, + }, + { + name: "only status has changed and previous reconciliation is in progress", etcdStatusChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateProcessing), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "only status has changed and previous reconciliation is completed", + etcdStatusChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateSucceeded), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, @@ -173,7 +199,16 @@ func TestBuildPredicateWithNoAutoReconcileButReconcileAnnotPresent(t *testing.T) shouldAllowUpdateEvent: true, }, { - name: "neither spec nor status has changed", + name: "neither spec nor status has changed and previous reconciliation is in error", + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateError), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "neither spec nor status has changed and previous reconciliation is completed", + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateSucceeded), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, @@ -186,7 +221,7 @@ func TestBuildPredicateWithNoAutoReconcileButReconcileAnnotPresent(t *testing.T) predicate := r.buildPredicate() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, true) + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, tc.lastOperationState, true) g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) @@ -196,16 +231,7 @@ func TestBuildPredicateWithNoAutoReconcileButReconcileAnnotPresent(t *testing.T) } func TestBuildPredicateWithAutoReconcileAndReconcileAnnotSet(t *testing.T) { - testCases := []struct { - name string - etcdSpecChanged bool - etcdStatusChanged bool - // expected behavior for different event types - shouldAllowCreateEvent bool - shouldAllowDeleteEvent bool - shouldAllowGenericEvent bool - shouldAllowUpdateEvent bool - }{ + testCases := []predicateTestCase{ { name: "only spec has changed", etcdSpecChanged: true, @@ -215,8 +241,18 @@ func TestBuildPredicateWithAutoReconcileAndReconcileAnnotSet(t *testing.T) { shouldAllowUpdateEvent: true, }, { - name: "only status has changed", + name: "only status has changed and previous reconciliation is in progress", etcdStatusChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateProcessing), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "only status has changed and previous reconciliation is completed", + etcdStatusChanged: true, + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateSucceeded), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, @@ -232,7 +268,24 @@ func TestBuildPredicateWithAutoReconcileAndReconcileAnnotSet(t *testing.T) { shouldAllowUpdateEvent: true, }, { - name: "neither spec nor status has changed", + name: "neither spec nor status has changed and previous reconciliation is in error", + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateError), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "neither spec nor status has changed and previous reconciliation is in progress", + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateProcessing), + shouldAllowCreateEvent: true, + shouldAllowDeleteEvent: true, + shouldAllowGenericEvent: false, + shouldAllowUpdateEvent: false, + }, + { + name: "neither spec nor status has changed and previous reconciliation is completed", + lastOperationState: utils.PointerOf(druidv1alpha1.LastOperationStateSucceeded), shouldAllowCreateEvent: true, shouldAllowDeleteEvent: true, shouldAllowGenericEvent: false, @@ -245,7 +298,7 @@ func TestBuildPredicateWithAutoReconcileAndReconcileAnnotSet(t *testing.T) { predicate := r.buildPredicate() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, true) + updatedEtcd := updateEtcd(etcd, tc.etcdSpecChanged, tc.etcdStatusChanged, tc.lastOperationState, true) g.Expect(predicate.Create(event.CreateEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowCreateEvent)) g.Expect(predicate.Delete(event.DeleteEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowDeleteEvent)) g.Expect(predicate.Generic(event.GenericEvent{Object: updatedEtcd})).To(Equal(tc.shouldAllowGenericEvent)) @@ -271,7 +324,7 @@ func createEtcd() *druidv1alpha1.Etcd { return etcd } -func updateEtcd(originalEtcd *druidv1alpha1.Etcd, specChanged, statusChanged, reconcileAnnotPresent bool) *druidv1alpha1.Etcd { +func updateEtcd(originalEtcd *druidv1alpha1.Etcd, specChanged, statusChanged bool, lastOpState *druidv1alpha1.LastOperationState, reconcileAnnotPresent bool) *druidv1alpha1.Etcd { newEtcd := originalEtcd.DeepCopy() annotations := make(map[string]string) if reconcileAnnotPresent { @@ -288,6 +341,12 @@ func updateEtcd(originalEtcd *druidv1alpha1.Etcd, specChanged, statusChanged, re newEtcd.Status.ReadyReplicas = 2 newEtcd.Status.Ready = pointer.Bool(false) } + if lastOpState != nil { + newEtcd.Status.LastOperation = &druidv1alpha1.LastOperation{ + Type: druidv1alpha1.LastOperationTypeReconcile, + State: *lastOpState, + } + } return newEtcd } diff --git a/internal/controller/utils/reconciler.go b/internal/controller/utils/reconciler.go index 22ec754a6..b87892bdc 100644 --- a/internal/controller/utils/reconciler.go +++ b/internal/controller/utils/reconciler.go @@ -79,6 +79,7 @@ func (r ReconcileStepResult) HasErrors() bool { return len(r.errs) > 0 } +// NeedsRequeue returns true if reconciler should requeue the request. func (r ReconcileStepResult) NeedsRequeue() bool { return r.HasErrors() || r.result.Requeue || r.result.RequeueAfter > 0 } diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index dfd225daa..b1a8afb95 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -41,7 +41,7 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul lease := &coordinationv1.Lease{} if err := r.cl.Get(ctx, kutil.Key(etcd.Namespace, leaseName), lease); err != nil { if apierrors.IsNotFound(err) { - r.logger.Error(fmt.Errorf("lease not found"), "name", leaseName) + r.logger.Error(fmt.Errorf("lease not found"), "lease not found", "name", leaseName) continue } r.logger.Error(err, "failed to get lease", "name", leaseName) From 4843c75c1f3f4cc8a45cf98639dd41c632b95ead Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Sun, 23 Jun 2024 23:39:39 +0530 Subject: [PATCH 234/235] Minor fixes --- api/v1alpha1/etcd.go | 2 +- charts/druid/templates/druid-clusterrole.yaml | 1 + internal/component/statefulset/statefulset.go | 2 +- internal/controller/etcd/reconcile_status.go | 7 ++++++- internal/health/condition/check_all_members.go | 15 ++++++++------- internal/health/etcdmember/check_ready.go | 11 ++++++++--- skaffold.yaml | 10 +++++----- 7 files changed, 30 insertions(+), 18 deletions(-) diff --git a/api/v1alpha1/etcd.go b/api/v1alpha1/etcd.go index 366282058..9d4c9e85e 100644 --- a/api/v1alpha1/etcd.go +++ b/api/v1alpha1/etcd.go @@ -44,7 +44,7 @@ const ( // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.spec.selector // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready` // +kubebuilder:printcolumn:name="Quorate",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="All Members Ready",type=string,JSONPath=`.status.conditions[?(@.type=="AllMembersReady")].status` diff --git a/charts/druid/templates/druid-clusterrole.yaml b/charts/druid/templates/druid-clusterrole.yaml index 886b178da..87aeac0ff 100644 --- a/charts/druid/templates/druid-clusterrole.yaml +++ b/charts/druid/templates/druid-clusterrole.yaml @@ -147,3 +147,4 @@ rules: - create - patch - update + - delete diff --git a/internal/component/statefulset/statefulset.go b/internal/component/statefulset/statefulset.go index 7165d8c92..78d9a2420 100644 --- a/internal/component/statefulset/statefulset.go +++ b/internal/component/statefulset/statefulset.go @@ -206,7 +206,7 @@ func (r _resource) createOrPatchWithReplicas(ctx component.OperatorContext, etcd fmt.Sprintf("Error creating or patching StatefulSet: %s for etcd: %v", desiredStatefulSet.Name, druidv1alpha1.GetNamespaceName(etcd.ObjectMeta))) } - ctx.Logger.Info("triggered creation of statefulSet", "statefulSet", getObjectKey(etcd.ObjectMeta), "operationResult", opResult) + ctx.Logger.Info("triggered create/patch of statefulSet", "statefulSet", getObjectKey(etcd.ObjectMeta), "operationResult", opResult) return nil } diff --git a/internal/controller/etcd/reconcile_status.go b/internal/controller/etcd/reconcile_status.go index 8a2457fe6..94ece8f7b 100644 --- a/internal/controller/etcd/reconcile_status.go +++ b/internal/controller/etcd/reconcile_status.go @@ -61,7 +61,12 @@ func (r *Reconciler) inspectStatefulSetAndMutateETCDStatus(ctx component.Operato Kind: sts.Kind, Name: sts.Name, } - ready, _ := utils.IsStatefulSetReady(etcd.Spec.Replicas, sts) + expectedReplicas := etcd.Spec.Replicas + // if the latest Etcd spec has not yet been reconciled by druid, then check sts readiness against sts.spec.replicas instead + if etcd.Status.ObservedGeneration == nil || *etcd.Status.ObservedGeneration != etcd.Generation { + expectedReplicas = *sts.Spec.Replicas + } + ready, _ := utils.IsStatefulSetReady(expectedReplicas, sts) etcd.Status.CurrentReplicas = sts.Status.CurrentReplicas etcd.Status.ReadyReplicas = sts.Status.ReadyReplicas etcd.Status.UpdatedReplicas = sts.Status.UpdatedReplicas diff --git a/internal/health/condition/check_all_members.go b/internal/health/condition/check_all_members.go index 0b72c9f7a..0e07a2fca 100644 --- a/internal/health/condition/check_all_members.go +++ b/internal/health/condition/check_all_members.go @@ -23,16 +23,17 @@ func (a *allMembersReady) Check(_ context.Context, etcd druidv1alpha1.Etcd) Resu } } - result := &result{ + res := &result{ conType: druidv1alpha1.ConditionTypeAllMembersReady, status: druidv1alpha1.ConditionFalse, reason: "NotAllMembersReady", message: "At least one member is not ready", } - if int32(len(etcd.Status.Members)) < etcd.Spec.Replicas { + if int32(len(etcd.Status.Members)) < etcd.Spec.Replicas && + etcd.Status.ObservedGeneration != nil && *etcd.Status.ObservedGeneration == etcd.Generation { // not all members are registered yet - return result + return res } // If we are here this means that all members have registered. Check if any member @@ -45,12 +46,12 @@ func (a *allMembersReady) Check(_ context.Context, etcd druidv1alpha1.Etcd) Resu } } if ready { - result.status = druidv1alpha1.ConditionTrue - result.reason = "AllMembersReady" - result.message = "All members are ready" + res.status = druidv1alpha1.ConditionTrue + res.reason = "AllMembersReady" + res.message = "All members are ready" } - return result + return res } // AllMembersReadyCheck returns a check for the "AllMembersReady" condition. diff --git a/internal/health/etcdmember/check_ready.go b/internal/health/etcdmember/check_ready.go index b1a8afb95..6300ee729 100644 --- a/internal/health/etcdmember/check_ready.go +++ b/internal/health/etcdmember/check_ready.go @@ -40,11 +40,16 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul for _, leaseName := range leaseNames { lease := &coordinationv1.Lease{} if err := r.cl.Get(ctx, kutil.Key(etcd.Namespace, leaseName), lease); err != nil { - if apierrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { + r.logger.Error(err, "failed to get lease", "name", leaseName) + } + // If latest Etcd spec has been reconciled, then all expected leases should have been created by now. + // An error is logged for such not-found member leases. + if etcd.Status.ObservedGeneration != nil && *etcd.Status.ObservedGeneration == etcd.Generation { r.logger.Error(fmt.Errorf("lease not found"), "lease not found", "name", leaseName) - continue } - r.logger.Error(err, "failed to get lease", "name", leaseName) + // In cases where Etcd.spec.replicas has increased, but the latest Etcd spec has not been reconciled by druid, + // the leases for new etcd members may not have been created yet. Such not-found member leases are ignored. continue } leases = append(leases, lease) diff --git a/skaffold.yaml b/skaffold.yaml index 7d7761821..e7051f9d9 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -24,11 +24,8 @@ deploy: chartPath: charts/druid namespace: default skipBuildDependencies: true - setValues: - controllers: - compaction: - etcdEventsThreshold: 15 - metricsScrapeWaitDuration: 30s + setValues: # empty `setValues` value is not allowed + foo: bar profiles: - name: e2e-test activation: @@ -39,6 +36,9 @@ profiles: value: etcd: etcdStatusSyncPeriod: 5s + compaction: + etcdEventsThreshold: 15 + metricsScrapeWaitDuration: 30s - name: enable-etcdcomponents-webhook activation: - env: "DRUID_ENABLE_ETCD_COMPONENTS_WEBHOOK=true" From 1411202205045feef87af3325e007104db1fc132 Mon Sep 17 00:00:00 2001 From: Shreyas Rao Date: Mon, 24 Jun 2024 12:33:08 +0530 Subject: [PATCH 235/235] Remove scale subresource label selector; fix compaction e2e test --- api/v1alpha1/etcd.go | 2 +- .../crd-druid.gardener.cloud_etcds.yaml | 1 - .../bases/crd-druid.gardener.cloud_etcds.yaml | 1 - test/e2e/etcd_compaction_test.go | 38 +++++++++++++------ 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/etcd.go b/api/v1alpha1/etcd.go index 9d4c9e85e..5b3a2cfed 100644 --- a/api/v1alpha1/etcd.go +++ b/api/v1alpha1/etcd.go @@ -44,7 +44,7 @@ const ( // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.spec.selector +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready` // +kubebuilder:printcolumn:name="Quorate",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="All Members Ready",type=string,JSONPath=`.status.conditions[?(@.type=="AllMembersReady")].status` diff --git a/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml index cb3524e17..eec9d9c9f 100644 --- a/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml @@ -1970,7 +1970,6 @@ spec: storage: true subresources: scale: - labelSelectorPath: .status.labelSelector specReplicasPath: .spec.replicas statusReplicasPath: .status.replicas status: {} diff --git a/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml index cb3524e17..eec9d9c9f 100644 --- a/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml @@ -1970,7 +1970,6 @@ spec: storage: true subresources: scale: - labelSelectorPath: .status.labelSelector specReplicasPath: .spec.replicas statusReplicasPath: .status.replicas status: {} diff --git a/test/e2e/etcd_compaction_test.go b/test/e2e/etcd_compaction_test.go index 7d41b2dde..96c08fb8d 100644 --- a/test/e2e/etcd_compaction_test.go +++ b/test/e2e/etcd_compaction_test.go @@ -7,15 +7,16 @@ package e2e import ( "context" "fmt" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "time" + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" druidstore "github.com/gardener/etcd-druid/internal/store" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -133,35 +134,50 @@ var _ = Describe("Etcd Compaction", func() { _, err = triggerOnDemandSnapshot(ctx, kubeconfigPath, namespace, etcdName, debugPod.Name, debugPod.Spec.Containers[0].Name, 8080, brtypes.SnapshotKindDelta) Expect(err).ShouldNot(HaveOccurred()) + latestSnapshotsAfterPopulate, err = getLatestSnapshots(ctx, kubeconfigPath, namespace, etcdName, debugPod.Name, debugPod.Spec.Containers[0].Name, 8080) + Expect(err).ShouldNot(HaveOccurred()) + latestSnapshotAfterPopulate = latestSnapshotsAfterPopulate.FullSnapshot + if numDeltas := len(latestSnapshotsAfterPopulate.DeltaSnapshots); numDeltas > 0 { + latestSnapshotAfterPopulate = latestSnapshotsAfterPopulate.DeltaSnapshots[numDeltas-1] + } + logger.Info("waiting for compaction job to become successful") + // Cannot check job status since it immediately gets deleted by compaction controller + // after successful completion. Hence, we check the snapshots before checking the compaction job status. Eventually(func() error { ctx, cancelFunc := context.WithTimeout(context.Background(), singleNodeEtcdTimeout) defer cancelFunc() + latestSnapshotsAfterCompaction, err := getLatestSnapshots(ctx, kubeconfigPath, namespace, etcdName, debugPod.Name, debugPod.Spec.Containers[0].Name, 8080) + if err != nil { + return fmt.Errorf("failed to get latest snapshots: %w", err) + } + if len(latestSnapshotsAfterCompaction.DeltaSnapshots) != 0 { + return fmt.Errorf("latest delta snapshot count is not 0") + } + if !latestSnapshotsAfterCompaction.FullSnapshot.CreatedOn.After(latestSnapshotAfterPopulate.CreatedOn) || + latestSnapshotsAfterCompaction.FullSnapshot.LastRevision != latestSnapshotAfterPopulate.LastRevision { + return fmt.Errorf("compaction is not yet successful") + } + req := types.NamespacedName{ Name: druidv1alpha1.GetCompactionJobName(etcd.ObjectMeta), Namespace: etcd.Namespace, } - j := &batchv1.Job{} if err := cl.Get(ctx, req, j); err != nil { + if apierrors.IsNotFound(err) { + return nil + } return err } - if j.Status.Succeeded < 1 { return fmt.Errorf("compaction job started but not yet successful") } - return nil }, singleNodeEtcdTimeout, pollingInterval).Should(BeNil()) logger.Info("compaction job is successful") - By("Verify that all the delta snapshots are compacted to full snapshots by compaction triggerred at first 15th revision") - latestSnapshotsAfterPopulate, err = getLatestSnapshots(ctx, kubeconfigPath, namespace, etcdName, debugPod.Name, debugPod.Spec.Containers[0].Name, 8080) - Expect(err).ShouldNot(HaveOccurred()) - - Expect(len(latestSnapshotsAfterPopulate.DeltaSnapshots)).Should(BeNumerically("==", 0)) - By("Put additional data into etcd") logger.Info("populating etcd with sequential key-value pairs", "fromKey", fmt.Sprintf("%s-16", etcdKeyPrefix), "fromValue", fmt.Sprintf("%s-16", etcdValuePrefix), @@ -170,7 +186,7 @@ var _ = Describe("Etcd Compaction", func() { err = populateEtcd(ctx, logger, kubeconfigPath, namespace, etcdName, debugPod.Name, debugPod.Spec.Containers[0].Name, etcdKeyPrefix, etcdValuePrefix, 16, 20, time.Second*1) Expect(err).ShouldNot(HaveOccurred()) - By("Trigger on-demand delta snapshot") + By("Trigger next on-demand delta snapshot") _, err = triggerOnDemandSnapshot(ctx, kubeconfigPath, namespace, etcdName, debugPod.Name, debugPod.Spec.Containers[0].Name, 8080, brtypes.SnapshotKindDelta) Expect(err).ShouldNot(HaveOccurred())