From 4f100b6de785dc602d1e216dad3d8fff98f6aa13 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 1 Mar 2021 21:02:35 +0100 Subject: [PATCH 1/6] store the backupconfig as json struct --- api/v1/postgres_types.go | 92 +++++++++++++++++------------- api/v1/zz_generated.deepcopy.go | 15 +++++ controllers/postgres_controller.go | 23 ++++++-- go.sum | 1 + 4 files changed, 84 insertions(+), 47 deletions(-) diff --git a/api/v1/postgres_types.go b/api/v1/postgres_types.go index 7de5b0c9..68badc0a 100644 --- a/api/v1/postgres_types.go +++ b/api/v1/postgres_types.go @@ -28,57 +28,67 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Version",type=string,JSONPath=`.spec.version` -// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.description` -// +kubebuilder:printcolumn:name="Load-Balancer-IP",type=string,JSONPath=`.status.socket.ip` -// +kubebuilder:printcolumn:name="Load-Balancer-Port",type=integer,JSONPath=`.status.socket.port` - -// UIDLabelName Name of the label referencing the owning Postgres resource in the control cluster -const UIDLabelName string = "postgres.database.fits.cloud/uuid" - -// TenantLabelName Name of the tenant label -const TenantLabelName string = "postgres.database.fits.cloud/tenant" - -// ProjectIDLabelName Name of the ProjectID label -const ProjectIDLabelName string = "postgres.database.fits.cloud/project-id" - -// ManagedByLabelName Name of the managed-by label -const ManagedByLabelName string = "postgres.database.fits.cloud/managed-by" - -// ManagedByLabelValue Value of the managed-by label -const ManagedByLabelValue string = "postgreslet" - -// PostgresFinalizerName Name of the finalizer to use -const PostgresFinalizerName string = "postgres.finalizers.database.fits.cloud" - -// Backup configure parametes of the database backup -const ( - // S3URL defines the s3 endpoint URL for backup - BackupSecretS3Endpoint = "s3Endpoint" - // S3BucketName defines the name of the S3 bucket for backup - BackupSecretS3BucketName = "s3BucketName" - // Retention defines how many days a backup will persist - BackupSecretRetention = "retention" - // Schedule defines how often a backup should be made, in cron format - BackupSecretSchedule = "schedule" - BackupSecretAccessKey = "accesskey" - BackupSecretSecretKey = "secretkey" - BackupSecretProjectKey = "project" -) - const ( + // UIDLabelName Name of the label referencing the owning Postgres resource in the control cluster + UIDLabelName string = "postgres.database.fits.cloud/uuid" + // TenantLabelName Name of the tenant label + TenantLabelName string = "postgres.database.fits.cloud/tenant" + // ProjectIDLabelName Name of the ProjectID label + ProjectIDLabelName string = "postgres.database.fits.cloud/project-id" + // ManagedByLabelName Name of the managed-by label + ManagedByLabelName string = "postgres.database.fits.cloud/managed-by" + // ManagedByLabelValue Value of the managed-by label + ManagedByLabelValue string = "postgreslet" + // PostgresFinalizerName Name of the finalizer to use + PostgresFinalizerName string = "postgres.finalizers.database.fits.cloud" + // BackupConfigKey defines the key under which the BackupConfig is stored in the data map. + BackupConfigKey = "config" + + // Sun shortcut for Sunday Sun Weekday = iota + // Mon Monday Mon + // Tue Tuesday Tue + // Wed Wednesday Wed + // Thu Thusday Thu + // Fri Friday Fri + // Sat Saturday Sat + // All every day All ) +// BackupConfig defines all properties to configure backup of a database. +// This config is stored in the data section under the key BackupConfigKey as json payload. +type BackupConfig struct { + // Retention defines how many versions should be held in s3 + Retention string `json:"retention"` + // Schedule in cron syntax when to run the backup periodically + Schedule string `json:"schedule"` + + // S3Endpoint the url of the s3 endpoint + S3Endpoint string `json:"s3endpoint"` + // S3BucketName is the name of the bucket where the backup should be stored. + S3BucketName string `json:"s3bucketname"` + // S3Region the region of the aws s3 + S3Region string `json:"s3region"` + // S3AccessKey is the accesskey which must have write access + S3AccessKey string `json:"s3accesskey"` + // S3SecretKey is the secretkey which must match to the accesskey + S3SecretKey string `json:"s3secretkey"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Version",type=string,JSONPath=`.spec.version` +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.description` +// +kubebuilder:printcolumn:name="Load-Balancer-IP",type=string,JSONPath=`.status.socket.ip` +// +kubebuilder:printcolumn:name="Load-Balancer-Port",type=integer,JSONPath=`.status.socket.port` + // Postgres is the Schema for the postgres API type Postgres struct { metav1.TypeMeta `json:",inline"` @@ -307,7 +317,7 @@ func (p *Postgres) ToPeripheralResourceName() string { return p.generateTeamID() + "-" + p.generateDatabaseName() } -// ToUserPasswordSecret returns the secret containing user password pairs +// ToUserPasswordsSecret returns the secret containing user password pairs func (p *Postgres) ToUserPasswordsSecret(src *corev1.SecretList, scheme *runtime.Scheme) (*corev1.Secret, error) { secret := &corev1.Secret{} secret.Namespace = p.Namespace diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 82972b32..0fa47592 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -34,6 +34,21 @@ func (in *AccessList) DeepCopy() *AccessList { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupConfig) DeepCopyInto(out *BackupConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupConfig. +func (in *BackupConfig) DeepCopy() *BackupConfig { + if in == nil { + return nil + } + out := new(BackupConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Maintenance) DeepCopyInto(out *Maintenance) { *out = *in diff --git a/controllers/postgres_controller.go b/controllers/postgres_controller.go index cf0b472e..8fc82536 100644 --- a/controllers/postgres_controller.go +++ b/controllers/postgres_controller.go @@ -8,6 +8,7 @@ package controllers import ( "context" + "encoding/json" "fmt" "net/url" "time" @@ -272,7 +273,17 @@ func (r *PostgresReconciler) createOrUpdateBackupConfig(ctx context.Context, p * return fmt.Errorf("error while getting the backup secret from control plane cluster: %w", err) } - s3url, err := url.Parse(string(backupSecret.Data[pg.BackupSecretS3Endpoint])) + backupConfigJSON, ok := backupSecret.Data[pg.BackupConfigKey] + if !ok { + return fmt.Errorf("no backupConfig stored in the secret") + } + var backupConfig pg.BackupConfig + err := json.Unmarshal(backupConfigJSON, &backupConfig) + if err != nil { + return fmt.Errorf("unable to unmarshal backupconfig:%w", err) + } + + s3url, err := url.Parse(backupConfig.S3Endpoint) if err != nil { return fmt.Errorf("error while parsing the s3 endpoint url in the backup secret: %w", err) } @@ -284,11 +295,11 @@ func (r *PostgresReconciler) createOrUpdateBackupConfig(ctx context.Context, p * walES3Endpoint := s3url.String() // use the rest as provided in the secret - bucketName := string(backupSecret.Data[pg.BackupSecretS3BucketName]) - awsAccessKeyID := string(backupSecret.Data[pg.BackupSecretAccessKey]) - awsSecretAccessKey := string(backupSecret.Data[pg.BackupSecretSecretKey]) - backupSchedule := string(backupSecret.Data[pg.BackupSecretSchedule]) - backupNumToRetain := string(backupSecret.Data[pg.BackupSecretRetention]) + bucketName := backupConfig.S3BucketName + awsAccessKeyID := backupConfig.S3AccessKey + awsSecretAccessKey := backupConfig.S3SecretKey + backupSchedule := backupConfig.Schedule + backupNumToRetain := backupConfig.Retention // create updated content for pod environment configmap data := map[string]string{ diff --git a/go.sum b/go.sum index 3a4e42c6..89363a2f 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5I github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= From 7587be184ee7ed52bb6c53260f55f44b9b0279ac Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Tue, 2 Mar 2021 07:30:53 +0100 Subject: [PATCH 2/6] Add backupconfig.name --- api/v1/postgres_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/v1/postgres_types.go b/api/v1/postgres_types.go index 68badc0a..802b1750 100644 --- a/api/v1/postgres_types.go +++ b/api/v1/postgres_types.go @@ -65,6 +65,8 @@ const ( // BackupConfig defines all properties to configure backup of a database. // This config is stored in the data section under the key BackupConfigKey as json payload. type BackupConfig struct { + // Name is a user defined description + Name string `json:"retention"` // Retention defines how many versions should be held in s3 Retention string `json:"retention"` // Schedule in cron syntax when to run the backup periodically From d07ee0dc98a22f8fa8eada97d2ac9e29149cb5d6 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Wed, 3 Mar 2021 07:40:20 +0100 Subject: [PATCH 3/6] add project and tenant --- api/v1/postgres_types.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/v1/postgres_types.go b/api/v1/postgres_types.go index 802b1750..7a63b00b 100644 --- a/api/v1/postgres_types.go +++ b/api/v1/postgres_types.go @@ -66,7 +66,11 @@ const ( // This config is stored in the data section under the key BackupConfigKey as json payload. type BackupConfig struct { // Name is a user defined description - Name string `json:"retention"` + Name string `json:"name"` + // ProjectID the project this backup is mapped to + ProjectID string `json:"project"` + // Tenant the tenant of the backup + Tenant string `json:"tenant"` // Retention defines how many versions should be held in s3 Retention string `json:"retention"` // Schedule in cron syntax when to run the backup periodically From fbc70b51054422c11c8fe08508fa49f2fdd78312 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Wed, 3 Mar 2021 14:56:39 +0100 Subject: [PATCH 4/6] add id --- api/v1/postgres_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/v1/postgres_types.go b/api/v1/postgres_types.go index 7a63b00b..fd80c453 100644 --- a/api/v1/postgres_types.go +++ b/api/v1/postgres_types.go @@ -65,6 +65,8 @@ const ( // BackupConfig defines all properties to configure backup of a database. // This config is stored in the data section under the key BackupConfigKey as json payload. type BackupConfig struct { + // ID of this backupConfig + ID string `json:"id"` // Name is a user defined description Name string `json:"name"` // ProjectID the project this backup is mapped to From f8cfbb037ff49b902be4a4d1d198202c427b43ce Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 4 Mar 2021 07:46:49 +0100 Subject: [PATCH 5/6] add extra labels --- api/v1/postgres_types.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/v1/postgres_types.go b/api/v1/postgres_types.go index fd80c453..da4310fe 100644 --- a/api/v1/postgres_types.go +++ b/api/v1/postgres_types.go @@ -41,6 +41,10 @@ const ( ManagedByLabelValue string = "postgreslet" // PostgresFinalizerName Name of the finalizer to use PostgresFinalizerName string = "postgres.finalizers.database.fits.cloud" + // CreatedByAnnotationKey is used to store who in person created this database + CreatedByAnnotationKey string = "postgres.database.fits.cloud/created-by" + // BackupConfigLabelName if set to true, this secret stores the backupConfig + BackupConfigLabelName string = "postgres.database.fits.cloud/is-backup" // BackupConfigKey defines the key under which the BackupConfig is stored in the data map. BackupConfigKey = "config" From 82462a6948cc46774c7a8c22dff46973d4280645 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 4 Mar 2021 08:26:55 +0100 Subject: [PATCH 6/6] remove unused --- api/v1/postgres_types.go | 26 -------------------------- api/v1/zz_generated.deepcopy.go | 17 ----------------- 2 files changed, 43 deletions(-) diff --git a/api/v1/postgres_types.go b/api/v1/postgres_types.go index d22c9c96..7026df40 100644 --- a/api/v1/postgres_types.go +++ b/api/v1/postgres_types.go @@ -48,23 +48,6 @@ const ( BackupConfigLabelName string = "postgres.database.fits.cloud/is-backup" // BackupConfigKey defines the key under which the BackupConfig is stored in the data map. BackupConfigKey = "config" - - // Sun shortcut for Sunday - Sun Weekday = iota - // Mon Monday - Mon - // Tue Tuesday - Tue - // Wed Wednesday - Wed - // Thu Thusday - Thu - // Fri Friday - Fri - // Sat Saturday - Sat - // All every day - All ) // BackupConfig defines all properties to configure backup of a database. @@ -176,15 +159,6 @@ type Size struct { StorageSize string `json:"storageSize,omitempty"` } -// Weekday defines a weekday or everyday -type Weekday int - -// TimeWindow defines an interval in time -type TimeWindow struct { - Start metav1.Time `json:"start,omitempty"` - End metav1.Time `json:"end,omitempty"` -} - // PostgresStatus defines the observed state of Postgres type PostgresStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 0a0433a7..fb793c29 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -183,20 +183,3 @@ func (in *Socket) DeepCopy() *Socket { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TimeWindow) DeepCopyInto(out *TimeWindow) { - *out = *in - in.Start.DeepCopyInto(&out.Start) - in.End.DeepCopyInto(&out.End) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeWindow. -func (in *TimeWindow) DeepCopy() *TimeWindow { - if in == nil { - return nil - } - out := new(TimeWindow) - in.DeepCopyInto(out) - return out -}