From 653085038a48daf090f3d473794472fb23f1d529 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 14 Apr 2022 10:48:11 +0000 Subject: [PATCH] [Feature] BackupInProgress & Maintenace Conditions --- CHANGELOG.md | 1 + pkg/apis/deployment/v1/conditions.go | 6 + pkg/apis/deployment/v2alpha1/conditions.go | 6 + pkg/deployment/agency/state.go | 24 +++- pkg/deployment/agency/target.go | 2 +- .../reconcile/action_maintenance_condition.go | 2 +- .../reconcile/action_resign_leadership.go | 4 +- pkg/deployment/reconcile/plan_builder_high.go | 4 +- .../reconcile/plan_builder_maintenance.go | 128 ++++++++++++++++++ .../reconcile/plan_builder_utils.go | 1 + 10 files changed, 168 insertions(+), 10 deletions(-) create mode 100644 pkg/deployment/reconcile/plan_builder_maintenance.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c160afea0..44fb9cf02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - (Bugfix) ArangoSync port fix - (Bugfix) Fix GetClient lock system - (Feature) Backup InProgress Agency key discovery +- (Feature) Backup & Maintenance Conditions ## [1.2.9](https://github.com/arangodb/kube-arangodb/tree/1.2.9) (2022-03-30) - (Feature) Improve Kubernetes clientsets management diff --git a/pkg/apis/deployment/v1/conditions.go b/pkg/apis/deployment/v1/conditions.go index d3d540621..12f871583 100644 --- a/pkg/apis/deployment/v1/conditions.go +++ b/pkg/apis/deployment/v1/conditions.go @@ -96,6 +96,12 @@ const ( // ConditionTypeLicenseSet indicates that license V2 is set on cluster. ConditionTypeLicenseSet ConditionType = "LicenseSet" + + // ConditionTypeBackupInProgress indicates that there is Backup in progress on cluster + ConditionTypeBackupInProgress ConditionType = "BackupInProgress" + + // ConditionTypeMaintenance indicates that maintenance is enabled on cluster + ConditionTypeMaintenance ConditionType = "Maintenance" ) // Condition represents one current condition of a deployment or deployment member. diff --git a/pkg/apis/deployment/v2alpha1/conditions.go b/pkg/apis/deployment/v2alpha1/conditions.go index 0de54f061..8a7769ae3 100644 --- a/pkg/apis/deployment/v2alpha1/conditions.go +++ b/pkg/apis/deployment/v2alpha1/conditions.go @@ -96,6 +96,12 @@ const ( // ConditionTypeLicenseSet indicates that license V2 is set on cluster. ConditionTypeLicenseSet ConditionType = "LicenseSet" + + // ConditionTypeBackupInProgress indicates that there is Backup in progress on cluster + ConditionTypeBackupInProgress ConditionType = "BackupInProgress" + + // ConditionTypeMaintenance indicates that maintenance is enabled on cluster + ConditionTypeMaintenance ConditionType = "Maintenance" ) // Condition represents one current condition of a deployment or deployment member. diff --git a/pkg/deployment/agency/state.go b/pkg/deployment/agency/state.go index f21effc01..aa74f267c 100644 --- a/pkg/deployment/agency/state.go +++ b/pkg/deployment/agency/state.go @@ -27,6 +27,7 @@ import ( "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" + "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/errors" ) @@ -113,18 +114,31 @@ type StateSupervision struct { Maintenance StateExists `json:"Maintenance,omitempty"` } -type StateExists bool +type StateExists []byte -func (d *StateExists) Exists() bool { +func (d StateExists) Hash() string { if d == nil { - return false + return "" } - return bool(*d) + return util.SHA256(d) +} + +func (d StateExists) Exists() bool { + return d != nil } func (d *StateExists) UnmarshalJSON(bytes []byte) error { - *d = bytes != nil + if bytes == nil { + *d = nil + return nil + } + + z := make([]byte, len(bytes)) + + copy(z, bytes) + + *d = z return nil } diff --git a/pkg/deployment/agency/target.go b/pkg/deployment/agency/target.go index 69e213dbb..89b291565 100644 --- a/pkg/deployment/agency/target.go +++ b/pkg/deployment/agency/target.go @@ -25,5 +25,5 @@ type StateTarget struct { } type StateTargetHotBackup struct { - Create *StateExists `json:"Create,omitempty"` + Create StateExists `json:"Create,omitempty"` } diff --git a/pkg/deployment/reconcile/action_maintenance_condition.go b/pkg/deployment/reconcile/action_maintenance_condition.go index 2680189db..0b706a0f8 100644 --- a/pkg/deployment/reconcile/action_maintenance_condition.go +++ b/pkg/deployment/reconcile/action_maintenance_condition.go @@ -60,7 +60,7 @@ func (a *actionSetMaintenanceCondition) Start(ctx context.Context) (bool, error) } else { if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { - if agencyState.Supervision.Maintenance { + if agencyState.Supervision.Maintenance.Exists() { return s.Conditions.Update(api.ConditionTypeMaintenanceMode, true, "Maintenance", "Maintenance enabled") } else { return s.Conditions.Remove(api.ConditionTypeMaintenanceMode) diff --git a/pkg/deployment/reconcile/action_resign_leadership.go b/pkg/deployment/reconcile/action_resign_leadership.go index f2c2cee45..2148552e9 100644 --- a/pkg/deployment/reconcile/action_resign_leadership.go +++ b/pkg/deployment/reconcile/action_resign_leadership.go @@ -80,7 +80,7 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK { log.Warn().Err(err).Msgf("Maintenance is enabled, skipping action") return true, errors.WithStack(err) - } else if agencyState.Supervision.Maintenance { + } else if agencyState.Supervision.Maintenance.Exists() { // We are done, action cannot be handled on maintenance mode log.Warn().Msgf("Maintenance is enabled, skipping action") return true, nil @@ -129,7 +129,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK { log.Error().Msgf("Unable to get maintenance mode") return false, false, nil - } else if agencyState.Supervision.Maintenance { + } else if agencyState.Supervision.Maintenance.Exists() { log.Warn().Msgf("Maintenance is enabled, skipping action") // We are done, action cannot be handled on maintenance mode m.CleanoutJobID = "" diff --git a/pkg/deployment/reconcile/plan_builder_high.go b/pkg/deployment/reconcile/plan_builder_high.go index b858026e7..4d9da85dd 100644 --- a/pkg/deployment/reconcile/plan_builder_high.go +++ b/pkg/deployment/reconcile/plan_builder_high.go @@ -58,7 +58,9 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A ApplyIfEmptyWithBackOff(LicenseCheck, 30*time.Second, updateClusterLicense). ApplyIfEmpty(createTopologyMemberConditionPlan). ApplyIfEmpty(createRebalancerCheckPlan). - ApplyWithBackOff(BackOffCheck, time.Minute, emptyPlanBuilder)) + ApplyWithBackOff(BackOffCheck, time.Minute, emptyPlanBuilder)). + Apply(createBackupInProgressConditionPlan). // Discover backups always + Apply(createMaintenanceConditionPlan) // Discover maintenance always return r.Plan(), r.BackOff(), true } diff --git a/pkg/deployment/reconcile/plan_builder_maintenance.go b/pkg/deployment/reconcile/plan_builder_maintenance.go new file mode 100644 index 000000000..6948c58c0 --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_maintenance.go @@ -0,0 +1,128 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" + "github.com/rs/zerolog" +) + +func createBackupInProgressConditionPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + + if spec.Mode.Get() != api.DeploymentModeCluster { + return nil + } + + cache, ok := context.GetAgencyCache() + if !ok { + return nil + } + + currentCondition, currentConditionExists := status.Conditions.Get(api.ConditionTypeBackupInProgress) + + backupInProgress := cache.Target.HotBackup.Create + + if currentConditionExists { + // Condition exists + if !backupInProgress.Exists() { + // Condition needs to be removed + return api.Plan{ + removeConditionActionV2("Backup not in progress", api.ConditionTypeBackupInProgress), + } + } + + // Backup is in progress + + hash := backupInProgress.Hash() + + if !currentCondition.IsTrue() || currentCondition.Hash != hash { + return api.Plan{ + updateConditionActionV2("Backup in progress", api.ConditionTypeBackupInProgress, true, "Backup In Progress", "", hash), + } + } + + return nil + } else { + if backupInProgress.Exists() { + return api.Plan{ + updateConditionActionV2("Backup in progress", api.ConditionTypeBackupInProgress, true, "Backup In Progress", "", backupInProgress.Hash()), + } + } + + return nil + } +} + +func createMaintenanceConditionPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + + if spec.Mode.Get() != api.DeploymentModeCluster { + return nil + } + + cache, ok := context.GetAgencyCache() + if !ok { + return nil + } + + currentCondition, currentConditionExists := status.Conditions.Get(api.ConditionTypeMaintenance) + + backupInProgress := cache.Target.HotBackup.Create + + if currentConditionExists { + // Condition exists + if !backupInProgress.Exists() { + // Condition needs to be removed + return api.Plan{ + removeConditionActionV2("Backup not in progress", api.ConditionTypeMaintenance), + } + } + + // Backup is in progress + + hash := backupInProgress.Hash() + + if !currentCondition.IsTrue() || currentCondition.Hash != hash { + return api.Plan{ + updateConditionActionV2("Backup in progress", api.ConditionTypeMaintenance, true, "Backup In Progress", "", hash), + } + } + + return nil + } else { + if backupInProgress.Exists() { + return api.Plan{ + updateConditionActionV2("Backup in progress", api.ConditionTypeMaintenance, true, "Backup In Progress", "", backupInProgress.Hash()), + } + } + + return nil + } +} diff --git a/pkg/deployment/reconcile/plan_builder_utils.go b/pkg/deployment/reconcile/plan_builder_utils.go index 0640ef774..2937cdd70 100644 --- a/pkg/deployment/reconcile/plan_builder_utils.go +++ b/pkg/deployment/reconcile/plan_builder_utils.go @@ -77,6 +77,7 @@ func removeConditionActionV2(actionReason string, conditionType api.ConditionTyp AddParam(setConditionActionV2KeyType, setConditionActionV2KeyTypeRemove) } +//nolint:unparam func updateConditionActionV2(actionReason string, conditionType api.ConditionType, status bool, reason, message, hash string) api.Action { statusBool := core.ConditionTrue if !status {