From 65e5b08968754954be8f933dbe5bcd1e7c88928e Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Thu, 18 Feb 2021 13:26:40 +0000 Subject: [PATCH] Fix AKS Volume resize mode --- CHANGELOG.md | 1 + pkg/apis/deployment/v1/member_phase.go | 2 ++ pkg/apis/deployment/v2alpha1/member_phase.go | 2 ++ .../reconcile/action_rotate_start_member.go | 2 +- pkg/deployment/reconcile/plan_builder_storage.go | 14 ++++++++++++-- pkg/deployment/reconcile/timeouts.go | 2 +- pkg/deployment/resilience/member_failure.go | 2 +- pkg/deployment/resources/pod_inspector.go | 2 +- 8 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a50c0160..548d4e866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Change Log ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) +- Fix AKS Volume Resize mode ## [1.1.4](https://github.com/arangodb/kube-arangodb/tree/1.1.4) (2021-02-15) - Add support for spec.ClusterDomain to be able to use FQDN in ArangoDB cluster communication diff --git a/pkg/apis/deployment/v1/member_phase.go b/pkg/apis/deployment/v1/member_phase.go index 5f6a52725..531ef3941 100644 --- a/pkg/apis/deployment/v1/member_phase.go +++ b/pkg/apis/deployment/v1/member_phase.go @@ -42,6 +42,8 @@ const ( MemberPhaseShuttingDown MemberPhase = "ShuttingDown" // MemberPhaseRotating indicates that a member is being rotated MemberPhaseRotating MemberPhase = "Rotating" + // MemberPhaseRotateStart indicates that a member is being rotated but wont get up outside of plan + MemberPhaseRotateStart MemberPhase = "RotateStart" // MemberPhaseUpgrading indicates that a member is in the process of upgrading its database data format MemberPhaseUpgrading MemberPhase = "Upgrading" ) diff --git a/pkg/apis/deployment/v2alpha1/member_phase.go b/pkg/apis/deployment/v2alpha1/member_phase.go index aea7717bd..fbb8382ba 100644 --- a/pkg/apis/deployment/v2alpha1/member_phase.go +++ b/pkg/apis/deployment/v2alpha1/member_phase.go @@ -42,6 +42,8 @@ const ( MemberPhaseShuttingDown MemberPhase = "ShuttingDown" // MemberPhaseRotating indicates that a member is being rotated MemberPhaseRotating MemberPhase = "Rotating" + // MemberPhaseRotateStart indicates that a member is being rotated but wont get up outside of plan + MemberPhaseRotateStart MemberPhase = "RotateStart" // MemberPhaseUpgrading indicates that a member is in the process of upgrading its database data format MemberPhaseUpgrading MemberPhase = "Upgrading" ) diff --git a/pkg/deployment/reconcile/action_rotate_start_member.go b/pkg/deployment/reconcile/action_rotate_start_member.go index a68e7240c..8f7f708fc 100644 --- a/pkg/deployment/reconcile/action_rotate_start_member.go +++ b/pkg/deployment/reconcile/action_rotate_start_member.go @@ -92,7 +92,7 @@ func (a *actionRotateStartMember) Start(ctx context.Context) (bool, error) { } } // Update status - m.Phase = api.MemberPhaseRotating + m.Phase = api.MemberPhaseRotateStart if err := a.actionCtx.UpdateMember(m); err != nil { return false, errors.WithStack(err) diff --git a/pkg/deployment/reconcile/plan_builder_storage.go b/pkg/deployment/reconcile/plan_builder_storage.go index 17a54c260..ef936e15e 100644 --- a/pkg/deployment/reconcile/plan_builder_storage.go +++ b/pkg/deployment/reconcile/plan_builder_storage.go @@ -141,13 +141,23 @@ func pvcResizePlan(log zerolog.Logger, group api.ServerGroup, groupSpec api.Serv api.NewAction(api.ActionTypePVCResize, group, memberID), } case api.PVCResizeModeRotate: - return api.Plan{ + var plan api.Plan + + if group == api.ServerGroupDBServers { + plan = append(plan, + api.NewAction(api.ActionTypeCleanOutMember, group, memberID), + ) + } + + plan = append(plan, api.NewAction(api.ActionTypeRotateStartMember, group, memberID), api.NewAction(api.ActionTypePVCResize, group, memberID), api.NewAction(api.ActionTypePVCResized, group, memberID), api.NewAction(api.ActionTypeRotateStopMember, group, memberID), api.NewAction(api.ActionTypeWaitForMemberUp, group, memberID), - } + ) + + return plan default: log.Error().Str("server-group", group.AsRole()).Str("mode", mode.String()). Msg("Requested mode is not supported") diff --git a/pkg/deployment/reconcile/timeouts.go b/pkg/deployment/reconcile/timeouts.go index 51da82d2f..4ae4d1967 100644 --- a/pkg/deployment/reconcile/timeouts.go +++ b/pkg/deployment/reconcile/timeouts.go @@ -33,7 +33,7 @@ const ( renewTLSCACertificateTimeout = time.Minute * 30 operationTLSCACertificateTimeout = time.Minute * 30 rotateMemberTimeout = time.Minute * 15 - pvcResizeTimeout = time.Minute * 15 + pvcResizeTimeout = time.Minute * 30 pvcResizedTimeout = time.Minute * 15 backupRestoreTimeout = time.Minute * 15 shutdownMemberTimeout = time.Minute * 30 diff --git a/pkg/deployment/resilience/member_failure.go b/pkg/deployment/resilience/member_failure.go index 1774a9129..e1339b52e 100644 --- a/pkg/deployment/resilience/member_failure.go +++ b/pkg/deployment/resilience/member_failure.go @@ -56,7 +56,7 @@ func (r *Resilience) CheckMemberFailure() error { switch m.Phase { case api.MemberPhaseNone: continue - case api.MemberPhaseUpgrading, api.MemberPhaseRotating, api.MemberPhaseCleanOut: + case api.MemberPhaseUpgrading, api.MemberPhaseRotating, api.MemberPhaseCleanOut, api.MemberPhaseRotateStart: if len(status.Plan) == 0 { log.Error().Msgf("No plan but member is in phase %s - marking as failed", m.Phase) m.Phase = api.MemberPhaseFailed diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index d0a36bd96..e31353ec4 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -248,7 +248,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspector.Insp case api.MemberPhaseNone: // Do nothing log.Debug().Str("pod-name", podName).Msg("PodPhase is None, waiting for the pod to be recreated") - case api.MemberPhaseShuttingDown, api.MemberPhaseUpgrading, api.MemberPhaseFailed: + case api.MemberPhaseShuttingDown, api.MemberPhaseUpgrading, api.MemberPhaseFailed, api.MemberPhaseRotateStart: // Shutdown was intended, so not need to do anything here. // Just mark terminated wasTerminated := m.Conditions.IsTrue(api.ConditionTypeTerminated)