From 38eef8085e7492d908ed5469ba2dc8519e0697cc Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 21 Sep 2022 15:35:01 +0000 Subject: [PATCH 1/2] [Feature] ScaleDown Candidate --- CHANGELOG.md | 1 + pkg/apis/deployment/annotations.go | 13 +++--- pkg/apis/deployment/v1/conditions.go | 2 + pkg/apis/deployment/v2alpha1/conditions.go | 2 + pkg/deployment/member/phase_updates.go | 1 + pkg/deployment/reconcile/plan_builder_high.go | 1 + .../reconcile/plan_builder_scale.go | 45 ++++++++++++++++++- 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9ab1e17..6fd076ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - (Feature) Propagate sidecars' ports to a member's service - (Debug Package) Initial commit - (Feature) Detach PVC from deployment in Ordered indexing method +- (Feature) ScaleDown Candidate ## [1.2.16](https://github.com/arangodb/kube-arangodb/tree/1.2.16) (2022-09-14) - (Feature) Add ArangoDeployment ServerGroupStatus diff --git a/pkg/apis/deployment/annotations.go b/pkg/apis/deployment/annotations.go index 3cc8c7d69..782a2d57c 100644 --- a/pkg/apis/deployment/annotations.go +++ b/pkg/apis/deployment/annotations.go @@ -21,10 +21,11 @@ package deployment const ( - ArangoDeploymentAnnotationPrefix = "deployment.arangodb.com" - ArangoDeploymentPodMaintenanceAnnotation = ArangoDeploymentAnnotationPrefix + "/maintenance" - ArangoDeploymentPodRotateAnnotation = ArangoDeploymentAnnotationPrefix + "/rotate" - ArangoDeploymentPodReplaceAnnotation = ArangoDeploymentAnnotationPrefix + "/replace" - ArangoDeploymentPodDeleteNow = ArangoDeploymentAnnotationPrefix + "/delete_now" - ArangoDeploymentPlanCleanAnnotation = "plan." + ArangoDeploymentAnnotationPrefix + "/clean" + ArangoDeploymentAnnotationPrefix = "deployment.arangodb.com" + ArangoDeploymentPodMaintenanceAnnotation = ArangoDeploymentAnnotationPrefix + "/maintenance" + ArangoDeploymentPodRotateAnnotation = ArangoDeploymentAnnotationPrefix + "/rotate" + ArangoDeploymentPodReplaceAnnotation = ArangoDeploymentAnnotationPrefix + "/replace" + ArangoDeploymentPodDeleteNow = ArangoDeploymentAnnotationPrefix + "/delete_now" + ArangoDeploymentPodScaleDownCandidateAnnotation = ArangoDeploymentAnnotationPrefix + "/scale_down_candidate" + ArangoDeploymentPlanCleanAnnotation = "plan." + ArangoDeploymentAnnotationPrefix + "/clean" ) diff --git a/pkg/apis/deployment/v1/conditions.go b/pkg/apis/deployment/v1/conditions.go index 918aac8f0..34168efd0 100644 --- a/pkg/apis/deployment/v1/conditions.go +++ b/pkg/apis/deployment/v1/conditions.go @@ -68,6 +68,8 @@ const ( ConditionTypeSpecAccepted ConditionType = "SpecAccepted" // ConditionTypeMarkedToRemove indicates that the member is marked to be removed. ConditionTypeMarkedToRemove ConditionType = "MarkedToRemove" + // ConditionTypeScaleDownCandidate indicates that the member will be picked in ScaleDown operaion. + ConditionTypeScaleDownCandidate ConditionType = "ScaleDownCandidate" // ConditionTypeUpgradeFailed indicates that upgrade failed ConditionTypeUpgradeFailed ConditionType = "UpgradeFailed" diff --git a/pkg/apis/deployment/v2alpha1/conditions.go b/pkg/apis/deployment/v2alpha1/conditions.go index 4a01145a5..4ccd91fa2 100644 --- a/pkg/apis/deployment/v2alpha1/conditions.go +++ b/pkg/apis/deployment/v2alpha1/conditions.go @@ -68,6 +68,8 @@ const ( ConditionTypeSpecAccepted ConditionType = "SpecAccepted" // ConditionTypeMarkedToRemove indicates that the member is marked to be removed. ConditionTypeMarkedToRemove ConditionType = "MarkedToRemove" + // ConditionTypeScaleDownCandidate indicates that the member will be picked in ScaleDown operaion. + ConditionTypeScaleDownCandidate ConditionType = "ScaleDownCandidate" // ConditionTypeUpgradeFailed indicates that upgrade failed ConditionTypeUpgradeFailed ConditionType = "UpgradeFailed" diff --git a/pkg/deployment/member/phase_updates.go b/pkg/deployment/member/phase_updates.go index 0566adc82..8e2c6c952 100644 --- a/pkg/deployment/member/phase_updates.go +++ b/pkg/deployment/member/phase_updates.go @@ -100,6 +100,7 @@ func removeMemberConditionsMapFunc(m *api.MemberStatus) { m.Conditions.Remove(api.ConditionTypeTopologyAware) m.Conditions.Remove(api.MemberReplacementRequired) m.Conditions.Remove(api.ConditionTypePVCResizePending) + m.Conditions.Remove(api.ConditionTypeScaleDownCandidate) m.RemoveTerminationsBefore(time.Now().Add(-1 * recentTerminationsKeepPeriod)) diff --git a/pkg/deployment/reconcile/plan_builder_high.go b/pkg/deployment/reconcile/plan_builder_high.go index 3af8cbbc8..632c74d32 100644 --- a/pkg/deployment/reconcile/plan_builder_high.go +++ b/pkg/deployment/reconcile/plan_builder_high.go @@ -58,6 +58,7 @@ func (r *Reconciler) createHighPlan(ctx context.Context, apiObject k8sutil.APIOb ApplyIfEmpty(r.createTopologyMemberConditionPlan). ApplyIfEmpty(r.createRebalancerCheckPlan). ApplyIfEmpty(r.createMemberFailedRestoreHighPlan). + ApplyIfEmpty(r.scaleDownCandidate). ApplyWithBackOff(BackOffCheck, time.Minute, r.emptyPlanBuilder)). ApplyIfEmptyWithBackOff(TimezoneCheck, time.Minute, r.createTimezoneUpdatePlan). Apply(r.createBackupInProgressConditionPlan). // Discover backups always diff --git a/pkg/deployment/reconcile/plan_builder_scale.go b/pkg/deployment/reconcile/plan_builder_scale.go index a78226330..71304e103 100644 --- a/pkg/deployment/reconcile/plan_builder_scale.go +++ b/pkg/deployment/reconcile/plan_builder_scale.go @@ -23,6 +23,7 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/actions" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" @@ -86,7 +87,7 @@ func (r *Reconciler) createScalePlan(status api.DeploymentStatus, members api.Me Debug("Creating scale-up plan") } else if len(members) > count { // Note, we scale down 1 member at a time - if m, err := members.SelectMemberToRemove(getCleanedServer(context), topologyMissingMemberToRemoveSelector(status.Topology), topologyAwarenessMemberToRemoveSelector(group, status.Topology)); err != nil { + if m, err := members.SelectMemberToRemove(getCleanedServer(context), getScaleDownCandidates(), topologyMissingMemberToRemoveSelector(status.Topology), topologyAwarenessMemberToRemoveSelector(group, status.Topology)); err != nil { r.planLogger.Err(err).Str("role", group.AsRole()).Warn("Failed to select member to remove") } else { ready, message := groupReadyForRestart(context, status, m, group) @@ -173,3 +174,45 @@ func getCleanedServer(ctx reconciler.ArangoAgencyGet) api.MemberToRemoveSelector return "", nil } } + +func getScaleDownCandidates() api.MemberToRemoveSelector { + return func(m api.MemberStatusList) (string, error) { + for _, member := range m { + if member.Conditions.IsTrue(api.ConditionTypeScaleDownCandidate) { + return member.ID, nil + } + } + return "", nil + } +} + +func (r *Reconciler) scaleDownCandidate(ctx context.Context, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + context PlanBuilderContext) api.Plan { + var plan api.Plan + + for _, m := range status.Members.AsList() { + cache, ok := context.ACS().ClusterCache(m.Member.ClusterID) + if !ok { + continue + } + pod, ok := cache.Pod().V1().GetSimple(m.Member.Pod.GetName()) + if !ok { + continue + } + + _, annotationExists := pod.Annotations[deployment.ArangoDeploymentPodScaleDownCandidateAnnotation] + + conditionExists := m.Member.Conditions.IsTrue(api.ConditionTypeScaleDownCandidate) + + if annotationExists != conditionExists { + if annotationExists { + plan = append(plan, updateMemberConditionActionV2("Marked as ScaleDownCandidate", api.ConditionTypeScaleDownCandidate, m.Group, m.Member.ID, true, "Marked as ScaleDownCandidate", "", "")) + } else { + plan = append(plan, removeMemberConditionActionV2("Unmarked as ScaleDownCandidate", api.ConditionTypeScaleDownCandidate, m.Group, m.Member.ID)) + } + } + } + + return plan +} From 3dc419168495207fe69a59222bc216634241025f Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 22 Sep 2022 08:06:14 +0000 Subject: [PATCH 2/2] Change logic --- pkg/apis/deployment/v1/member_status_list.go | 5 ++++ .../deployment/v2alpha1/member_status_list.go | 5 ++++ pkg/deployment/member/phase_updates.go | 1 - .../reconcile/plan_builder_scale.go | 28 +++++++++---------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/pkg/apis/deployment/v1/member_status_list.go b/pkg/apis/deployment/v1/member_status_list.go index 1ac5c5714..3f71c6fb3 100644 --- a/pkg/apis/deployment/v1/member_status_list.go +++ b/pkg/apis/deployment/v1/member_status_list.go @@ -149,6 +149,11 @@ func (l MemberStatusList) SelectMemberToRemove(selectors ...MemberToRemoveSelect return m, nil } } + for _, m := range l { + if m.Conditions.IsTrue(ConditionTypeScaleDownCandidate) { + return m, nil + } + } // Try to find a not ready member for _, m := range l { if m.Phase.IsPending() { diff --git a/pkg/apis/deployment/v2alpha1/member_status_list.go b/pkg/apis/deployment/v2alpha1/member_status_list.go index 0ce58bcfb..2dd4a5064 100644 --- a/pkg/apis/deployment/v2alpha1/member_status_list.go +++ b/pkg/apis/deployment/v2alpha1/member_status_list.go @@ -149,6 +149,11 @@ func (l MemberStatusList) SelectMemberToRemove(selectors ...MemberToRemoveSelect return m, nil } } + for _, m := range l { + if m.Conditions.IsTrue(ConditionTypeScaleDownCandidate) { + return m, nil + } + } // Try to find a not ready member for _, m := range l { if m.Phase.IsPending() { diff --git a/pkg/deployment/member/phase_updates.go b/pkg/deployment/member/phase_updates.go index 8e2c6c952..0566adc82 100644 --- a/pkg/deployment/member/phase_updates.go +++ b/pkg/deployment/member/phase_updates.go @@ -100,7 +100,6 @@ func removeMemberConditionsMapFunc(m *api.MemberStatus) { m.Conditions.Remove(api.ConditionTypeTopologyAware) m.Conditions.Remove(api.MemberReplacementRequired) m.Conditions.Remove(api.ConditionTypePVCResizePending) - m.Conditions.Remove(api.ConditionTypeScaleDownCandidate) m.RemoveTerminationsBefore(time.Now().Add(-1 * recentTerminationsKeepPeriod)) diff --git a/pkg/deployment/reconcile/plan_builder_scale.go b/pkg/deployment/reconcile/plan_builder_scale.go index 71304e103..4f6d98fc3 100644 --- a/pkg/deployment/reconcile/plan_builder_scale.go +++ b/pkg/deployment/reconcile/plan_builder_scale.go @@ -87,7 +87,7 @@ func (r *Reconciler) createScalePlan(status api.DeploymentStatus, members api.Me Debug("Creating scale-up plan") } else if len(members) > count { // Note, we scale down 1 member at a time - if m, err := members.SelectMemberToRemove(getCleanedServer(context), getScaleDownCandidates(), topologyMissingMemberToRemoveSelector(status.Topology), topologyAwarenessMemberToRemoveSelector(group, status.Topology)); err != nil { + if m, err := members.SelectMemberToRemove(getCleanedServer(context), topologyMissingMemberToRemoveSelector(status.Topology), topologyAwarenessMemberToRemoveSelector(group, status.Topology)); err != nil { r.planLogger.Err(err).Str("role", group.AsRole()).Warn("Failed to select member to remove") } else { ready, message := groupReadyForRestart(context, status, m, group) @@ -175,17 +175,6 @@ func getCleanedServer(ctx reconciler.ArangoAgencyGet) api.MemberToRemoveSelector } } -func getScaleDownCandidates() api.MemberToRemoveSelector { - return func(m api.MemberStatusList) (string, error) { - for _, member := range m { - if member.Conditions.IsTrue(api.ConditionTypeScaleDownCandidate) { - return member.ID, nil - } - } - return "", nil - } -} - func (r *Reconciler) scaleDownCandidate(ctx context.Context, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) api.Plan { @@ -196,12 +185,23 @@ func (r *Reconciler) scaleDownCandidate(ctx context.Context, apiObject k8sutil.A if !ok { continue } - pod, ok := cache.Pod().V1().GetSimple(m.Member.Pod.GetName()) + + annotationExists := false + + am, ok := cache.ArangoMember().V1().GetSimple(m.Member.ArangoMemberName(context.GetName(), m.Group)) if !ok { continue } - _, annotationExists := pod.Annotations[deployment.ArangoDeploymentPodScaleDownCandidateAnnotation] + if _, ok := am.Annotations[deployment.ArangoDeploymentPodScaleDownCandidateAnnotation]; ok { + annotationExists = true + } + + if pod, ok := cache.Pod().V1().GetSimple(m.Member.Pod.GetName()); ok { + if _, ok := pod.Annotations[deployment.ArangoDeploymentPodScaleDownCandidateAnnotation]; ok { + annotationExists = true + } + } conditionExists := m.Member.Conditions.IsTrue(api.ConditionTypeScaleDownCandidate)