diff --git a/CHANGELOG.md b/CHANGELOG.md index cc5d586ca..63c45b5dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ # Change Log ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) -- Prevent deletion not known PVC's +- Prevent deletion of not known PVC's +- Move Restore as Plan ## [1.0.2](https://github.com/arangodb/kube-arangodb/tree/1.0.2) (2020-04-16) - Added additional checks in UpToDate condition diff --git a/pkg/apis/deployment/v1/plan.go b/pkg/apis/deployment/v1/plan.go index 4087d1b1a..9bf74b4ed 100644 --- a/pkg/apis/deployment/v1/plan.go +++ b/pkg/apis/deployment/v1/plan.go @@ -75,6 +75,10 @@ const ( ActionTypePVCResized ActionType = "PVCResized" // UpToDateUpdateResized define up to date annotation in spec UpToDateUpdate ActionType = "UpToDateUpdate" + // ActionTypeBackupRestore restore plan + ActionTypeBackupRestore ActionType = "BackupRestore" + // ActionTypeBackupRestoreClean restore plan + ActionTypeBackupRestoreClean ActionType = "BackupRestoreClean" ) const ( diff --git a/pkg/deployment/backup/handler.go b/pkg/deployment/backup/handler.go deleted file mode 100644 index 332c9f830..000000000 --- a/pkg/deployment/backup/handler.go +++ /dev/null @@ -1,134 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2020 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// -// Author Lars Maier -// - -package backup - -import ( - "context" - - "github.com/arangodb/go-driver" - backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/rs/zerolog" -) - -type Context interface { - // GetSpec returns the current specification of the deployment - GetSpec() api.DeploymentSpec - // GetStatus returns the current status of the deployment - GetStatus() (api.DeploymentStatus, int32) - // UpdateStatus replaces the status of the deployment with the given status and - // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error - // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), - // creating one if needed. - GetDatabaseClient(ctx context.Context) (driver.Client, error) - // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) -} - -type BackupHandler struct { - log zerolog.Logger - context Context -} - -func NewHandler(log zerolog.Logger, context Context) *BackupHandler { - return &BackupHandler{ - log: log, - context: context, - } -} - -func (b *BackupHandler) restoreFrom(backupName string) error { - ctx := context.Background() - dbc, err := b.context.GetDatabaseClient(ctx) - if err != nil { - return err - } - - backupResource, err := b.context.GetBackup(backupName) - if err != nil { - return err - } - - backupID := backupResource.Status.Backup.ID - - // trigger the actual restore - if err := dbc.Backup().Restore(ctx, driver.BackupID(backupID), nil); err != nil { - return err - } - - return nil -} - -func (b *BackupHandler) CheckRestore() error { - - spec := b.context.GetSpec() - status, version := b.context.GetStatus() - - if spec.HasRestoreFrom() { - // We have to trigger a restore operation - if status.Restore == nil || status.Restore.RequestedFrom != spec.GetRestoreFrom() { - // Prepare message that we are starting restore - result := &api.DeploymentRestoreResult{ - RequestedFrom: spec.GetRestoreFrom(), - } - - result.State = api.DeploymentRestoreStateRestoring - - for i := 0; i < 100; i++ { - status, version := b.context.GetStatus() - status.Restore = result - b.context.UpdateStatus(status, version) - } - - // Request restoring - err := b.restoreFrom(spec.GetRestoreFrom()) - - if err != nil { - result.State = api.DeploymentRestoreStateRestoreFailed - result.Message = err.Error() - } else { - result.State = api.DeploymentRestoreStateRestored - } - - // try to update the status - for i := 0; i < 100; i++ { - status, version := b.context.GetStatus() - status.Restore = result - b.context.UpdateStatus(status, version) - } - } - - return nil - } - - if status.Restore == nil { - return nil - } - - // Remove the restore entry from status - status.Restore = nil - b.context.UpdateStatus(status, version) - return nil - -} diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index e33722137..4e75d00b0 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -39,7 +39,6 @@ import ( "k8s.io/client-go/tools/record" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/arangodb/kube-arangodb/pkg/deployment/backup" "github.com/arangodb/kube-arangodb/pkg/deployment/chaos" "github.com/arangodb/kube-arangodb/pkg/deployment/reconcile" "github.com/arangodb/kube-arangodb/pkg/deployment/resilience" @@ -112,7 +111,6 @@ type Deployment struct { reconciler *reconcile.Reconciler resilience *resilience.Resilience resources *resources.Resources - backup *backup.BackupHandler chaosMonkey *chaos.Monkey syncClientCache client.ClientCache haveServiceMonitorCRD bool @@ -135,7 +133,6 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De d.reconciler = reconcile.NewReconciler(deps.Log, d) d.resilience = resilience.NewResilience(deps.Log, d) d.resources = resources.NewResources(deps.Log, d) - d.backup = backup.NewHandler(deps.Log, d) if d.status.last.AcceptedSpec == nil { // We've validated the spec, so let's use it from now. d.status.last.AcceptedSpec = apiObject.Spec.DeepCopy() diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 3a85974bb..6f6f3c10c 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -236,10 +236,6 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, errors.Wrapf(err, "Removed member cleanup failed") } - if err := d.backup.CheckRestore(); err != nil { - return minInspectionInterval, errors.Wrapf(err, "Restore operation failed") - } - // At the end of the inspect, we cleanup terminated pods. if x, err := d.resources.CleanupTerminatedPods(); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod cleanup failed") diff --git a/pkg/deployment/reconcile/action_backup_restore.go b/pkg/deployment/reconcile/action_backup_restore.go new file mode 100644 index 000000000..b9145ea75 --- /dev/null +++ b/pkg/deployment/reconcile/action_backup_restore.go @@ -0,0 +1,123 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/go-driver" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeBackupRestore, newBackupRestoreAction) +} + +func newBackupRestoreAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &actionBackupRestore{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, backupRestoreTimeout) + + return a +} + +// actionBackupRestore implements an BackupRestore. +type actionBackupRestore struct { + // actionImpl implement timeout and member id functions + actionImpl + + actionEmptyCheckProgress +} + +func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { + spec := a.actionCtx.GetSpec() + status := a.actionCtx.GetStatus() + + if spec.RestoreFrom == nil { + return true, nil + } + + if status.Restore != nil { + a.log.Warn().Msg("Backup restore status should not be nil") + return true, nil + } + + dbc, err := a.actionCtx.GetDatabaseClient(ctx) + if err != nil { + return false, err + } + + backupResource, err := a.actionCtx.GetBackup(*spec.RestoreFrom) + if err != nil { + a.log.Error().Err(err).Msg("Unable to find backup") + return true, nil + } + + if backupResource.Status.Backup == nil { + a.log.Error().Msg("Backup ID is not set") + return true, nil + } + + if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + result := &api.DeploymentRestoreResult{ + RequestedFrom: spec.GetRestoreFrom(), + } + + result.State = api.DeploymentRestoreStateRestoring + + s.Restore = result + + return true + }, true); err != nil { + return false, err + } + + restoreError := dbc.Backup().Restore(ctx, driver.BackupID(backupResource.Status.Backup.ID), nil) + if restoreError != nil { + a.log.Error().Err(restoreError).Msg("Restore failed") + } + + if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + result := &api.DeploymentRestoreResult{ + RequestedFrom: spec.GetRestoreFrom(), + } + + if restoreError != nil { + result.State = api.DeploymentRestoreStateRestoreFailed + result.Message = restoreError.Error() + } else { + result.State = api.DeploymentRestoreStateRestored + } + + s.Restore = result + + return true + }); err != nil { + a.log.Error().Err(err).Msg("Unable to ser restored state") + return false, err + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_backup_restore_clean.go b/pkg/deployment/reconcile/action_backup_restore_clean.go new file mode 100644 index 000000000..3958a2968 --- /dev/null +++ b/pkg/deployment/reconcile/action_backup_restore_clean.go @@ -0,0 +1,65 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeBackupRestoreClean, newBackupRestoreCleanAction) +} + +func newBackupRestoreCleanAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &actionBackupRestoreClean{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, backupRestoreTimeout) + + return a +} + +// actionBackupRestoreClean implements an BackupRestoreClean. +type actionBackupRestoreClean struct { + // actionImpl implement timeout and member id functions + actionImpl + + actionEmptyCheckProgress +} + +func (a actionBackupRestoreClean) Start(ctx context.Context) (bool, error) { + if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if s.Restore == nil { + return false + } + + s.Restore = nil + return true + }, true); err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index d12c52bf8..15e6ddc46 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -26,6 +26,8 @@ import ( "context" "fmt" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" + "github.com/arangodb/go-driver/agency" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" v1 "k8s.io/api/core/v1" @@ -110,6 +112,8 @@ type ActionContext interface { InvalidateSyncStatus() // GetSpec returns a copy of the spec GetSpec() api.DeploymentSpec + // GetStatus returns a copy of the status + GetStatus() api.DeploymentStatus // DisableScalingCluster disables scaling DBservers and coordinators DisableScalingCluster() error // EnableScalingCluster enables scaling DBservers and coordinators @@ -117,6 +121,10 @@ type ActionContext interface { // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error SecretsInterface() k8sutil.SecretInterface + // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken + WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + // GetBackup receives information about a backup resource + GetBackup(backup string) (*backupApi.ArangoBackup, error) } // newActionContext creates a new ActionContext implementation. @@ -133,6 +141,22 @@ type actionContext struct { context Context } +func (ac *actionContext) GetStatus() api.DeploymentStatus { + a, _ := ac.context.GetStatus() + + s := a.DeepCopy() + + return *s +} + +func (ac *actionContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) { + return ac.context.GetBackup(backup) +} + +func (ac *actionContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { + return ac.context.WithStatusUpdate(action, force...) +} + func (ac *actionContext) SecretsInterface() k8sutil.SecretInterface { return ac.context.SecretsInterface() } diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index bb715831c..bd8fea770 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -25,6 +25,8 @@ package reconcile import ( "context" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" + "github.com/arangodb/arangosync-client/client" driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" @@ -115,4 +117,6 @@ type Context interface { WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface + // GetBackup receives information about a backup resource + GetBackup(backup string) (*backupApi.ArangoBackup, error) } diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 54f4fe93f..9ec3de6ea 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -232,6 +232,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb plan = createRotateTLSServerSNIPlan(ctx, log, spec, status, builderCtx) } + if plan.IsEmpty() { + plan = createRestorePlan(ctx, log, spec, status, builderCtx) + } + // Return plan return plan, true } diff --git a/pkg/deployment/reconcile/plan_builder_context.go b/pkg/deployment/reconcile/plan_builder_context.go index 2bba83f1e..92ec0ea7c 100644 --- a/pkg/deployment/reconcile/plan_builder_context.go +++ b/pkg/deployment/reconcile/plan_builder_context.go @@ -25,6 +25,8 @@ package reconcile import ( "context" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" + "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -61,6 +63,8 @@ type PlanBuilderContext interface { GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface + // GetBackup receives information about a backup resource + GetBackup(backup string) (*backupApi.ArangoBackup, error) } // newPlanBuilderContext creates a PlanBuilderContext from the given context diff --git a/pkg/deployment/reconcile/plan_builder_restore.go b/pkg/deployment/reconcile/plan_builder_restore.go new file mode 100644 index 000000000..442a29bb4 --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_restore.go @@ -0,0 +1,57 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func createRestorePlan(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, builderCtx PlanBuilderContext) api.Plan { + if spec.RestoreFrom == nil && status.Restore != nil { + return api.Plan{ + api.NewAction(api.ActionTypeBackupRestoreClean, api.ServerGroupUnknown, ""), + } + } + + if spec.RestoreFrom != nil && status.Restore == nil { + backup, err := builderCtx.GetBackup(spec.GetRestoreFrom()) + if err != nil { + log.Warn().Err(err).Msg("Backup not found") + return nil + } + + if backup.Status.Backup == nil { + log.Warn().Msg("Backup not yet ready") + return nil + } + + return api.Plan{ + api.NewAction(api.ActionTypeBackupRestore, api.ServerGroupUnknown, ""), + } + } + + return nil +} diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index 535d0faa7..be5abdbd5 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -29,6 +29,8 @@ import ( "io/ioutil" "testing" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" + "github.com/arangodb/arangosync-client/client" "github.com/arangodb/go-driver/agency" "github.com/rs/zerolog" @@ -55,6 +57,10 @@ type testContext struct { RecordedEvent *k8sutil.Event } +func (c *testContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) { + panic("implement me") +} + func (c *testContext) SecretsInterface() k8sutil.SecretInterface { panic("implement me") } diff --git a/pkg/deployment/reconcile/timeouts.go b/pkg/deployment/reconcile/timeouts.go index 7462c7c8b..002f6b38d 100644 --- a/pkg/deployment/reconcile/timeouts.go +++ b/pkg/deployment/reconcile/timeouts.go @@ -34,6 +34,7 @@ const ( rotateMemberTimeout = time.Minute * 15 pvcResizeTimeout = time.Minute * 15 pvcResizedTimeout = time.Minute * 15 + backupRestoreTimeout = time.Minute * 15 shutdownMemberTimeout = time.Minute * 30 upgradeMemberTimeout = time.Hour * 6 waitForMemberUpTimeout = time.Minute * 15 diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index de023730d..519a8250d 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -25,6 +25,8 @@ package resources import ( "context" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" + driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -94,4 +96,6 @@ type Context interface { GetAgency(ctx context.Context) (agency.Agency, error) // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + // GetBackup receives information about a backup resource + GetBackup(backup string) (*backupApi.ArangoBackup, error) }