diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f13ea058..37fac17cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - (Feature) Backup & Maintenance Conditions - (Bugfix) Disable member removal in case of health failure - (Bugfix) Reorder Topology management plan steps +- (Feature) UpdateInProgress & UpgradeInProgress Conditions ## [1.2.9](https://github.com/arangodb/kube-arangodb/tree/1.2.9) (2022-03-30) - (Feature) Improve Kubernetes clientsets management diff --git a/pkg/apis/deployment/v1/conditions.go b/pkg/apis/deployment/v1/conditions.go index 12f871583..38142284d 100644 --- a/pkg/apis/deployment/v1/conditions.go +++ b/pkg/apis/deployment/v1/conditions.go @@ -99,6 +99,10 @@ const ( // ConditionTypeBackupInProgress indicates that there is Backup in progress on cluster ConditionTypeBackupInProgress ConditionType = "BackupInProgress" + // ConditionTypeUpgradeInProgress indicates that there is upgrade in progress on cluster + ConditionTypeUpgradeInProgress ConditionType = "UpgradeInProgress" + // ConditionTypeUpdateInProgress indicates that there is update in progress on cluster + ConditionTypeUpdateInProgress ConditionType = "UpdateInProgress" // ConditionTypeMaintenance indicates that maintenance is enabled on cluster ConditionTypeMaintenance ConditionType = "Maintenance" diff --git a/pkg/apis/deployment/v2alpha1/conditions.go b/pkg/apis/deployment/v2alpha1/conditions.go index 8a7769ae3..2ca1335a6 100644 --- a/pkg/apis/deployment/v2alpha1/conditions.go +++ b/pkg/apis/deployment/v2alpha1/conditions.go @@ -99,6 +99,10 @@ const ( // ConditionTypeBackupInProgress indicates that there is Backup in progress on cluster ConditionTypeBackupInProgress ConditionType = "BackupInProgress" + // ConditionTypeUpgradeInProgress indicates that there is upgrade in progress on cluster + ConditionTypeUpgradeInProgress ConditionType = "UpgradeInProgress" + // ConditionTypeUpdateInProgress indicates that there is update in progress on cluster + ConditionTypeUpdateInProgress ConditionType = "UpdateInProgress" // ConditionTypeMaintenance indicates that maintenance is enabled on cluster ConditionTypeMaintenance ConditionType = "Maintenance" diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index 1ac79e4e5..d86ee4370 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -23,8 +23,6 @@ package reconcile import ( "context" - "github.com/arangodb/kube-arangodb/pkg/deployment/rotation" - "github.com/arangodb/kube-arangodb/pkg/deployment/features" "github.com/arangodb/kube-arangodb/pkg/deployment/resources" @@ -37,6 +35,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/actions" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" + "github.com/arangodb/kube-arangodb/pkg/deployment/rotation" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" "github.com/rs/zerolog" @@ -123,118 +122,170 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API decision := createRotateOrUpgradeDecision(log, spec, status, context) if decision.IsUpgrade() { + return createUpgradePlanInternalCondition(log, apiObject, spec, status, context, decision) + } else if decision.IsUpdate() { + return createUpdatePlanInternalCondition(log, apiObject, spec, status, cachedStatus, decision) + } else { + upgradeCondition := status.Conditions.IsTrue(api.ConditionTypeUpgradeInProgress) + updateCondition := status.Conditions.IsTrue(api.ConditionTypeUpdateInProgress) - for _, m := range status.Members.AsList() { - // Pre-check - d := decision[m.Member.ID] - if !d.upgrade { - continue - } + if upgradeCondition || updateCondition { + p := make(api.Plan, 0, 2) - // We have member to upgrade - if d.upgradeDecision.Hold { - // Holding upgrade - continue + if upgradeCondition { + p = append(p, removeConditionActionV2("Upgrade done", api.ConditionTypeUpgradeInProgress)) } - if !d.upgradeDecision.UpgradeAllowed { - context.CreateEvent(k8sutil.NewUpgradeNotAllowedEvent(apiObject, d.upgradeDecision.FromVersion, d.upgradeDecision.ToVersion, d.upgradeDecision.FromLicense, d.upgradeDecision.ToLicense)) - return nil, false + if updateCondition { + p = append(p, removeConditionActionV2("Update done", api.ConditionTypeUpdateInProgress)) } + + return p, false } + } - // Upgrade phase - // During upgrade always get first member which needs to be upgraded - for _, m := range status.Members.AsList() { - d := decision[m.Member.ID] - if !d.upgrade { - continue - } + return nil, false +} - // We have member to upgrade - if d.upgradeDecision.Hold { - // Holding upgrade - return nil, false - } +func createUpdatePlanInternalCondition(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, decision updateUpgradeDecisionMap) (api.Plan, bool) { + plan, idle := createUpdatePlanInternal(log, apiObject, spec, status, cachedStatus, decision) - if !d.upgradeDecision.UpgradeNeeded { - // In upgrade scenario but upgrade is not needed - return nil, false - } + if idle || len(plan) > 0 { + if !status.Conditions.IsTrue(api.ConditionTypeUpdateInProgress) { + plan = append(api.Plan{ + updateConditionActionV2("Update in progress", api.ConditionTypeUpdateInProgress, true, "", "", ""), + }, plan...) + } + } - if !d.upgradeDecision.UpgradeAllowed { - context.CreateEvent(k8sutil.NewUpgradeNotAllowedEvent(apiObject, d.upgradeDecision.FromVersion, d.upgradeDecision.ToVersion, d.upgradeDecision.FromLicense, d.upgradeDecision.ToLicense)) - return nil, false - } + return plan, idle +} - if d.updateAllowed { - // We are fine, group is alive so we can proceed - log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Upgrade allowed") - return createUpgradeMemberPlan(log, m.Member, m.Group, "Version upgrade", spec, status, !d.upgradeDecision.AutoUpgradeNeeded), false - } else if d.unsafeUpdateAllowed { - log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs upgrade but cluster is not ready. Either some shards are not in sync or some member is not ready, but unsafe upgrade is allowed") - return createUpgradeMemberPlan(log, m.Member, m.Group, "Version upgrade", spec, status, !d.upgradeDecision.AutoUpgradeNeeded), false - } else { - log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs upgrade but cluster is not ready. Either some shards are not in sync or some member is not ready.") - return nil, true - } +func createUpdatePlanInternal(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, decision updateUpgradeDecisionMap) (api.Plan, bool) { + // Update phase + for _, m := range status.Members.AsList() { + d := decision[m.Member.ID] + if !d.update { + continue } - log.Warn().Msg("Pod upgrade plan has been made, but it has been dropped due to missing flag") - return nil, false - } else if decision.IsUpdate() { - // Update phase - for _, m := range status.Members.AsList() { - d := decision[m.Member.ID] - if !d.update { + if !d.updateAllowed { + // Update is not allowed due to constraint + if !d.unsafeUpdateAllowed { + log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs restart but cluster is not ready. Either some shards are not in sync or some member is not ready.") continue } + log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs restart but cluster is not ready. Either some shards are not in sync or some member is not ready, but unsafe upgrade is allowed") + } - if !d.updateAllowed { - // Update is not allowed due to constraint - if !d.unsafeUpdateAllowed { - log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs restart but cluster is not ready. Either some shards are not in sync or some member is not ready.") - continue - } - log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs restart but cluster is not ready. Either some shards are not in sync or some member is not ready, but unsafe upgrade is allowed") - } + if m.Member.Conditions.IsTrue(api.ConditionTypeRestart) { + return createRotateMemberPlan(log, m.Member, m.Group, spec, "Restart flag present"), false + } + arangoMember, ok := cachedStatus.ArangoMember().V1().GetSimple(m.Member.ArangoMemberName(apiObject.GetName(), m.Group)) + if !ok { + continue + } - if m.Member.Conditions.IsTrue(api.ConditionTypeRestart) { - return createRotateMemberPlan(log, m.Member, m.Group, spec, "Restart flag present"), false - } - arangoMember, ok := cachedStatus.ArangoMember().V1().GetSimple(m.Member.ArangoMemberName(apiObject.GetName(), m.Group)) - if !ok { - continue - } + p, ok := cachedStatus.Pod().V1().GetSimple(m.Member.PodName) + if !ok { + p = nil + } - p, ok := cachedStatus.Pod().V1().GetSimple(m.Member.PodName) - if !ok { - p = nil - } + if mode, p, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, m.Member, m.Group, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil { + log.Err(err).Str("member", m.Member.ID).Msgf("Error while generating update plan") + continue + } else if mode != rotation.InPlaceRotation { + return api.Plan{actions.NewAction(api.ActionTypeSetMemberCondition, m.Group, m.Member, "Cleaning update"). + AddParam(api.ConditionTypePendingUpdate.String(), ""). + AddParam(api.ConditionTypeUpdating.String(), "T")}, false + } else { + p = p.After( + actions.NewAction(api.ActionTypeWaitForMemberUp, m.Group, m.Member), + actions.NewAction(api.ActionTypeWaitForMemberInSync, m.Group, m.Member)) + + p = p.Wrap(actions.NewAction(api.ActionTypeSetMemberCondition, m.Group, m.Member, reason). + AddParam(api.ConditionTypePendingUpdate.String(), "").AddParam(api.ConditionTypeUpdating.String(), "T"), + actions.NewAction(api.ActionTypeSetMemberCondition, m.Group, m.Member, reason). + AddParam(api.ConditionTypeUpdating.String(), "")) + + return p, false + } + } + return nil, true +} - if mode, p, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, m.Member, m.Group, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil { - log.Err(err).Str("member", m.Member.ID).Msgf("Error while generating update plan") - continue - } else if mode != rotation.InPlaceRotation { - return api.Plan{actions.NewAction(api.ActionTypeSetMemberCondition, m.Group, m.Member, "Cleaning update"). - AddParam(api.ConditionTypePendingUpdate.String(), ""). - AddParam(api.ConditionTypeUpdating.String(), "T")}, false - } else { - p = p.After( - actions.NewAction(api.ActionTypeWaitForMemberUp, m.Group, m.Member), - actions.NewAction(api.ActionTypeWaitForMemberInSync, m.Group, m.Member)) - - p = p.Wrap(actions.NewAction(api.ActionTypeSetMemberCondition, m.Group, m.Member, reason). - AddParam(api.ConditionTypePendingUpdate.String(), "").AddParam(api.ConditionTypeUpdating.String(), "T"), - actions.NewAction(api.ActionTypeSetMemberCondition, m.Group, m.Member, reason). - AddParam(api.ConditionTypeUpdating.String(), "")) - - return p, false - } +func createUpgradePlanInternalCondition(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext, decision updateUpgradeDecisionMap) (api.Plan, bool) { + plan, idle := createUpgradePlanInternal(log, apiObject, spec, status, context, decision) + + if idle || len(plan) > 0 { + if !status.Conditions.IsTrue(api.ConditionTypeUpgradeInProgress) { + plan = append(api.Plan{ + updateConditionActionV2("Upgrade in progress", api.ConditionTypeUpgradeInProgress, true, "", "", ""), + }, plan...) + } + } + + return plan, idle +} + +func createUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext, decision updateUpgradeDecisionMap) (api.Plan, bool) { + for _, m := range status.Members.AsList() { + // Pre-check + d := decision[m.Member.ID] + if !d.upgrade { + continue + } + + // We have member to upgrade + if d.upgradeDecision.Hold { + // Holding upgrade + continue + } + + if !d.upgradeDecision.UpgradeAllowed { + context.CreateEvent(k8sutil.NewUpgradeNotAllowedEvent(apiObject, d.upgradeDecision.FromVersion, d.upgradeDecision.ToVersion, d.upgradeDecision.FromLicense, d.upgradeDecision.ToLicense)) + return nil, false + } + } + + // Upgrade phase + // During upgrade always get first member which needs to be upgraded + for _, m := range status.Members.AsList() { + d := decision[m.Member.ID] + if !d.upgrade { + continue + } + + // We have member to upgrade + if d.upgradeDecision.Hold { + // Holding upgrade + return nil, false + } + + if !d.upgradeDecision.UpgradeNeeded { + // In upgrade scenario but upgrade is not needed + return nil, false + } + + if !d.upgradeDecision.UpgradeAllowed { + context.CreateEvent(k8sutil.NewUpgradeNotAllowedEvent(apiObject, d.upgradeDecision.FromVersion, d.upgradeDecision.ToVersion, d.upgradeDecision.FromLicense, d.upgradeDecision.ToLicense)) + return nil, false + } + + if d.updateAllowed { + // We are fine, group is alive so we can proceed + log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Upgrade allowed") + return createUpgradeMemberPlan(log, m.Member, m.Group, "Version upgrade", spec, status, !d.upgradeDecision.AutoUpgradeNeeded), false + } else if d.unsafeUpdateAllowed { + log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs upgrade but cluster is not ready. Either some shards are not in sync or some member is not ready, but unsafe upgrade is allowed") + return createUpgradeMemberPlan(log, m.Member, m.Group, "Version upgrade", spec, status, !d.upgradeDecision.AutoUpgradeNeeded), false + } else { + log.Info().Str("member", m.Member.ID).Str("Reason", d.updateMessage).Msg("Pod needs upgrade but cluster is not ready. Either some shards are not in sync or some member is not ready.") + return nil, true } - return nil, true } + log.Warn().Msg("Pod upgrade plan has been made, but it has been dropped due to missing flag") return nil, false }