From 7a72975c8196c34d43b26d53a0584a580162835e Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Fri, 27 Aug 2021 10:20:26 +0000 Subject: [PATCH] [Feature] Propagation modes --- CHANGELOG.md | 1 + pkg/apis/deployment/v1/conditions.go | 2 + .../v1/deployment_member_propagation_mode.go | 62 +++++++++ pkg/apis/deployment/v1/deployment_spec.go | 2 + .../deployment/v1/zz_generated.deepcopy.go | 5 + pkg/apis/deployment/v2alpha1/conditions.go | 2 + .../deployment_member_propagation_mode.go | 62 +++++++++ .../deployment/v2alpha1/deployment_spec.go | 2 + .../v2alpha1/zz_generated.deepcopy.go | 5 + pkg/deployment/deployment_inspector.go | 28 +++- pkg/deployment/reconcile/plan_builder_high.go | 121 +++++++++++------- .../reconcile/plan_builder_normal.go | 3 + .../reconcile/plan_builder_rotate_upgrade.go | 53 +------- pkg/deployment/reconcile/plan_builder_test.go | 7 +- pkg/deployment/reconcile/plan_builder_tls.go | 2 +- .../reconcile/plan_builder_tls_sni.go | 2 +- pkg/deployment/reconcile/plan_executor.go | 2 +- pkg/deployment/resources/pod_creator.go | 5 +- pkg/deployment/rotation/check.go | 119 +++++++++++++++++ 19 files changed, 378 insertions(+), 107 deletions(-) create mode 100644 pkg/apis/deployment/v1/deployment_member_propagation_mode.go create mode 100644 pkg/apis/deployment/v2alpha1/deployment_member_propagation_mode.go create mode 100644 pkg/deployment/rotation/check.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 77d599da6..58184c68f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Add Ephemeral Volumes for apps feature - Check if the DB server is cleaned out. - Render Pod Template in ArangoMember Spec and Status +- Add Pod PropagationModes ## [1.2.1](https://github.com/arangodb/kube-arangodb/tree/1.2.1) (2021-07-28) - Fix ArangoMember race with multiple ArangoDeployments within single namespace diff --git a/pkg/apis/deployment/v1/conditions.go b/pkg/apis/deployment/v1/conditions.go index a02bf6c2f..5f1d9d471 100644 --- a/pkg/apis/deployment/v1/conditions.go +++ b/pkg/apis/deployment/v1/conditions.go @@ -73,6 +73,8 @@ const ( ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode" // ConditionTypePendingRestart indicates that restart is required ConditionTypePendingRestart ConditionType = "PendingRestart" + // ConditionTypeRestart indicates that restart will be started + ConditionTypeRestart ConditionType = "Restart" // ConditionTypePendingTLSRotation indicates that TLS rotation is pending ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation" ) diff --git a/pkg/apis/deployment/v1/deployment_member_propagation_mode.go b/pkg/apis/deployment/v1/deployment_member_propagation_mode.go new file mode 100644 index 000000000..fcbaacc3d --- /dev/null +++ b/pkg/apis/deployment/v1/deployment_member_propagation_mode.go @@ -0,0 +1,62 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package v1 + +type DeploymentMemberPropagationMode string + +func (d *DeploymentMemberPropagationMode) Get() DeploymentMemberPropagationMode { + if d == nil { + return DeploymentMemberPropagationModeDefault + } + + return *d +} + +func (d DeploymentMemberPropagationMode) New() *DeploymentMemberPropagationMode { + return &d +} + +func (d DeploymentMemberPropagationMode) String() string { + return string(d) +} + +func (d *DeploymentMemberPropagationMode) Equal(b *DeploymentMemberPropagationMode) bool { + if d == nil && b == nil { + return true + } + + if d == nil || b == nil { + return false + } + + return *d == *b +} + +const ( + // DeploymentMemberPropagationModeDefault Define default propagation mode + DeploymentMemberPropagationModeDefault = DeploymentMemberPropagationModeAlways + // DeploymentMemberPropagationModeAlways define mode which restart member whenever change in pod is discovered + DeploymentMemberPropagationModeAlways DeploymentMemberPropagationMode = "always" + // DeploymentMemberPropagationModeOnRestart propagate member spec whenever pod is restarted. Do not restart member by default + DeploymentMemberPropagationModeOnRestart DeploymentMemberPropagationMode = "on-restart" +) diff --git a/pkg/apis/deployment/v1/deployment_spec.go b/pkg/apis/deployment/v1/deployment_spec.go index 85cbf3283..26e889060 100644 --- a/pkg/apis/deployment/v1/deployment_spec.go +++ b/pkg/apis/deployment/v1/deployment_spec.go @@ -149,6 +149,8 @@ type DeploymentSpec struct { SyncMasters ServerGroupSpec `json:"syncmasters"` SyncWorkers ServerGroupSpec `json:"syncworkers"` + MemberPropagationMode *DeploymentMemberPropagationMode `json:"memberPropagationMode,omitempty"` + Chaos ChaosSpec `json:"chaos"` Recovery *ArangoDeploymentRecoverySpec `json:"recovery,omitempty"` diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index 837c21381..328b667f0 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -583,6 +583,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.Coordinators.DeepCopyInto(&out.Coordinators) in.SyncMasters.DeepCopyInto(&out.SyncMasters) in.SyncWorkers.DeepCopyInto(&out.SyncWorkers) + if in.MemberPropagationMode != nil { + in, out := &in.MemberPropagationMode, &out.MemberPropagationMode + *out = new(DeploymentMemberPropagationMode) + **out = **in + } in.Chaos.DeepCopyInto(&out.Chaos) if in.Recovery != nil { in, out := &in.Recovery, &out.Recovery diff --git a/pkg/apis/deployment/v2alpha1/conditions.go b/pkg/apis/deployment/v2alpha1/conditions.go index 0dbd791c4..fcb14b847 100644 --- a/pkg/apis/deployment/v2alpha1/conditions.go +++ b/pkg/apis/deployment/v2alpha1/conditions.go @@ -73,6 +73,8 @@ const ( ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode" // ConditionTypePendingRestart indicates that restart is required ConditionTypePendingRestart ConditionType = "PendingRestart" + // ConditionTypeRestart indicates that restart will be started + ConditionTypeRestart ConditionType = "Restart" // ConditionTypePendingTLSRotation indicates that TLS rotation is pending ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation" ) diff --git a/pkg/apis/deployment/v2alpha1/deployment_member_propagation_mode.go b/pkg/apis/deployment/v2alpha1/deployment_member_propagation_mode.go new file mode 100644 index 000000000..ae1489c2d --- /dev/null +++ b/pkg/apis/deployment/v2alpha1/deployment_member_propagation_mode.go @@ -0,0 +1,62 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package v2alpha1 + +type DeploymentMemberPropagationMode string + +func (d *DeploymentMemberPropagationMode) Get() DeploymentMemberPropagationMode { + if d == nil { + return DeploymentMemberPropagationModeDefault + } + + return *d +} + +func (d DeploymentMemberPropagationMode) New() *DeploymentMemberPropagationMode { + return &d +} + +func (d DeploymentMemberPropagationMode) String() string { + return string(d) +} + +func (d *DeploymentMemberPropagationMode) Equal(b *DeploymentMemberPropagationMode) bool { + if d == nil && b == nil { + return true + } + + if d == nil || b == nil { + return false + } + + return *d == *b +} + +const ( + // DeploymentMemberPropagationModeDefault Define default propagation mode + DeploymentMemberPropagationModeDefault = DeploymentMemberPropagationModeAlways + // DeploymentMemberPropagationModeAlways define mode which restart member whenever change in pod is discovered + DeploymentMemberPropagationModeAlways DeploymentMemberPropagationMode = "always" + // DeploymentMemberPropagationModeOnRestart propagate member spec whenever pod is restarted. Do not restart member by default + DeploymentMemberPropagationModeOnRestart DeploymentMemberPropagationMode = "on-restart" +) diff --git a/pkg/apis/deployment/v2alpha1/deployment_spec.go b/pkg/apis/deployment/v2alpha1/deployment_spec.go index a2e5dff77..5fa054f32 100644 --- a/pkg/apis/deployment/v2alpha1/deployment_spec.go +++ b/pkg/apis/deployment/v2alpha1/deployment_spec.go @@ -149,6 +149,8 @@ type DeploymentSpec struct { SyncMasters ServerGroupSpec `json:"syncmasters"` SyncWorkers ServerGroupSpec `json:"syncworkers"` + MemberPropagationMode *DeploymentMemberPropagationMode `json:"memberPropagationMode,omitempty"` + Chaos ChaosSpec `json:"chaos"` Recovery *ArangoDeploymentRecoverySpec `json:"recovery,omitempty"` diff --git a/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go b/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go index 200866a92..7119ed028 100644 --- a/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go @@ -583,6 +583,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.Coordinators.DeepCopyInto(&out.Coordinators) in.SyncMasters.DeepCopyInto(&out.SyncMasters) in.SyncWorkers.DeepCopyInto(&out.SyncWorkers) + if in.MemberPropagationMode != nil { + in, out := &in.MemberPropagationMode, &out.MemberPropagationMode + *out = new(DeploymentMemberPropagationMode) + **out = **in + } in.Chaos.DeepCopyInto(&out.Chaos) if in.Recovery != nil { in, out := &in.Recovery, &out.Recovery diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 884b2b5d3..27ebf530f 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -303,15 +303,17 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, nil } else if status.AppliedVersion == checksum { - if !d.apiObject.Status.IsPlanEmpty() && status.Conditions.IsTrue(api.ConditionTypeUpToDate) { - if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil { + isUpToDate, reason := d.isUpToDateStatus() + + if !isUpToDate && status.Conditions.IsTrue(api.ConditionTypeUpToDate) { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, reason, "There are pending operations in plan or members are in restart process"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } return minInspectionInterval, nil } - if d.apiObject.Status.IsPlanEmpty() && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) { + if isUpToDate && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) { if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -349,6 +351,26 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return } +func (d *Deployment) isUpToDateStatus() (upToDate bool, reason string) { + if !d.apiObject.Status.IsPlanEmpty() { + return false, "Plan is not empty" + } + + upToDate = true + + d.apiObject.Status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + for _, member := range list { + if member.Conditions.IsTrue(api.ConditionTypeRestart) || member.Conditions.IsTrue(api.ConditionTypePendingRestart) { + upToDate = false + reason = "Pending restarts on members" + } + } + return nil + }) + + return +} + func (d *Deployment) refreshMaintenanceTTL(ctx context.Context) { if d.apiObject.Spec.Mode.Get() == api.DeploymentModeSingle { return diff --git a/pkg/deployment/reconcile/plan_builder_high.go b/pkg/deployment/reconcile/plan_builder_high.go index 9ea47fc4e..ed00481c7 100644 --- a/pkg/deployment/reconcile/plan_builder_high.go +++ b/pkg/deployment/reconcile/plan_builder_high.go @@ -25,7 +25,8 @@ package reconcile import ( "context" - "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + "github.com/arangodb/kube-arangodb/pkg/deployment/rotation" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" @@ -57,6 +58,9 @@ func (d *Reconciler) CreateHighPlan(ctx context.Context, cachedStatus inspectorI for id := len(status.Plan); id < len(newPlan); id++ { action := newPlan[id] d.context.CreateEvent(k8sutil.NewPlanAppendEvent(apiObject, action.Type.String(), action.Group.AsRole(), action.MemberID, action.Reason)) + if r := action.Reason; r != "" { + d.log.Info().Str("Action", action.Type.String()).Str("Role", action.Group.AsRole()).Str("Member", action.MemberID).Str("Type", "High").Msgf(r) + } } status.HighPriorityPlan = newPlan @@ -83,7 +87,7 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A ApplyIfEmpty(updateMemberPodTemplateSpec). ApplyIfEmpty(updateMemberPhasePlan). ApplyIfEmpty(createCleanOutPlan). - ApplyIfEmpty(updateMemberRotationFlag). + ApplyIfEmpty(updateMemberRotationConditionsPlan). Plan(), true } @@ -118,11 +122,16 @@ func updateMemberPhasePlan(ctx context.Context, status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { for _, m := range list { if m.Phase == api.MemberPhaseNone { - plan = append(plan, + p := api.Plan{ api.NewAction(api.ActionTypeMemberRIDUpdate, group, m.ID, "Regenerate member RID"), - api.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, m.ID, "Propagating status of pod"), - api.NewAction(api.ActionTypeMemberPhaseUpdate, group, m.ID, - "Move to Pending phase").AddParam(ActionTypeMemberPhaseUpdatePhaseKey, api.MemberPhasePending.String())) + } + + p = append(p, api.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, m.ID, "Propagating status of pod")) + + p = append(p, api.NewAction(api.ActionTypeMemberPhaseUpdate, group, m.ID, + "Move to Pending phase").AddParam(ActionTypeMemberPhaseUpdatePhaseKey, api.MemberPhasePending.String())) + + plan = append(plan, p...) } } @@ -132,66 +141,88 @@ func updateMemberPhasePlan(ctx context.Context, return plan } -func updateMemberRotationFlag(ctx context.Context, +func pendingRestartMemberConditionAction(group api.ServerGroup, memberID string, reason string) api.Action { + return api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingRestart.String(), "T") +} + +func restartMemberConditionAction(group api.ServerGroup, memberID string, reason string) api.Action { + return pendingRestartMemberConditionAction(group, memberID, reason).AddParam(api.ConditionTypeRestart.String(), "T") +} + +func tlsRotateConditionAction(group api.ServerGroup, memberID string, reason string) api.Action { + return api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingTLSRotation.String(), "T") +} + +func updateMemberRotationConditionsPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { var plan api.Plan - status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { for _, m := range list { - p, found := cachedStatus.Pod(m.PodName) - if !found { - continue + p, ok := cachedStatus.Pod(m.PodName) + if !ok { + p = nil } - if required, reason := updateMemberRotationFlagConditionCheck(log, apiObject, spec, cachedStatus, m, group, p); required { - log.Info().Msgf(reason) - plan = append(plan, restartMemberConditionPlan(group, m.ID, reason)...) + if p, err := updateMemberRotationConditions(log, apiObject, spec, cachedStatus, m, group, p); err != nil { + return err + } else if len(p) > 0 { + plan = append(plan, p...) } } return nil - }) + }); err != nil { + log.Err(err).Msgf("Error while generating rotation plan") + return nil + } return plan } -func restartMemberConditionPlan(group api.ServerGroup, memberID string, reason string) api.Plan { - return api.Plan{ - api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingRestart.String(), "T"), +func updateMemberRotationConditions(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, cachedStatus inspectorInterface.Inspector, member api.MemberStatus, group api.ServerGroup, p *core.Pod) (api.Plan, error) { + if member.Conditions.IsTrue(api.ConditionTypeRestart) { + return nil, nil } -} - -func tlsRotateConditionPlan(group api.ServerGroup, memberID string, reason string) api.Plan { - return api.Plan{ - api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingTLSRotation.String(), "T"), - } -} -func updateMemberRotationFlagConditionCheck(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, cachedStatus inspectorInterface.Inspector, m api.MemberStatus, group api.ServerGroup, p *core.Pod) (bool, string) { - if m.Conditions.IsTrue(api.ConditionTypePendingRestart) { - return false, "" + arangoMember, ok := cachedStatus.ArangoMember(member.ArangoMemberName(apiObject.GetName(), group)) + if !ok { + return nil, nil } - if m.Conditions.IsTrue(api.ConditionTypePendingTLSRotation) { - return true, "TLS Rotation pending" - } + if m, _, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, member, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil { + log.Error().Err(err).Msgf("Error while getting rotation details") + return nil, err + } else { + switch m { + case rotation.EnforcedRotation: + if reason != "" { + log.Info().Bool("enforced", true).Msgf(reason) + } else { + log.Info().Bool("enforced", true).Msgf("Unknown reason") + } + // We need to do enforced rotation + return api.Plan{restartMemberConditionAction(group, member.ID, reason)}, nil + case rotation.GracefulRotation, rotation.InPlaceRotation, rotation.SilentRotation: // TODO: Add support for InPlace and Silent rotation + if reason != "" { + log.Info().Bool("enforced", false).Msgf(reason) + } else { + log.Info().Bool("enforced", false).Msgf("Unknown reason") + } + // We need to do graceful rotation + if member.Conditions.IsTrue(api.ConditionTypePendingRestart) { + return nil, nil + } - pvc, exists := cachedStatus.PersistentVolumeClaim(m.PersistentVolumeClaimName) - if exists { - if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) { - return true, "PVC Resize pending" + if spec.MemberPropagationMode.Get() == api.DeploymentMemberPropagationModeAlways { + return api.Plan{restartMemberConditionAction(group, member.ID, reason)}, nil + } else { + return api.Plan{pendingRestartMemberConditionAction(group, member.ID, reason)}, nil + } + default: + return nil, nil } } - - if changed, reason := podNeedsRotation(log, apiObject, p, spec, group, m, cachedStatus); changed { - return true, reason - } - - if _, ok := p.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok { - return true, "Rotation flag present" - } - - return false, "" } diff --git a/pkg/deployment/reconcile/plan_builder_normal.go b/pkg/deployment/reconcile/plan_builder_normal.go index 8d700982e..a4cfe2c77 100644 --- a/pkg/deployment/reconcile/plan_builder_normal.go +++ b/pkg/deployment/reconcile/plan_builder_normal.go @@ -55,6 +55,9 @@ func (d *Reconciler) CreateNormalPlan(ctx context.Context, cachedStatus inspecto for id := len(status.Plan); id < len(newPlan); id++ { action := newPlan[id] d.context.CreateEvent(k8sutil.NewPlanAppendEvent(apiObject, action.Type.String(), action.Group.AsRole(), action.MemberID, action.Reason)) + if r := action.Reason; r != "" { + d.log.Info().Str("Action", action.Type.String()).Str("Role", action.Group.AsRole()).Str("Member", action.MemberID).Str("Type", "Normal").Msgf(r) + } } status.Plan = newPlan diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index f0616b11e..0a15ab8e6 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -27,8 +27,6 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/features" - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" - "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/go-driver" @@ -124,7 +122,7 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec, status, !decision.AutoUpgradeNeeded) } else { - if m.Conditions.IsTrue(api.ConditionTypePendingRestart) { + if m.Conditions.IsTrue(api.ConditionTypeRestart) { newPlan = createRotateMemberPlan(log, m, group, "Restart flag present") } } @@ -328,55 +326,6 @@ func arangoMemberPodTemplateNeedsUpdate(ctx context.Context, log zerolog.Logger, return "", false } -// podNeedsRotation returns true when the specification of the -// given pod differs from what it should be according to the -// given deployment spec. -// When true is returned, a reason for the rotation is already returned. -func podNeedsRotation(log zerolog.Logger, apiObject k8sutil.APIObject, p *core.Pod, spec api.DeploymentSpec, group api.ServerGroup, m api.MemberStatus, cachedStatus inspectorInterface.Inspector) (bool, string) { - if m.PodUID != p.UID { - return true, "Pod UID does not match, this pod is not managed by Operator. Recreating" - } - - if m.PodSpecVersion == "" { - return true, "Pod Spec Version is nil - recreating pod" - } - - member, ok := cachedStatus.ArangoMember(m.ArangoMemberName(apiObject.GetName(), group)) - if !ok { - return false, "" - } - - if member.Status.Template == nil { - return false, "" - } - - if member.Status.Template.RotationNeeded(member.Spec.Template) { - log.Info().Str("id", m.ID).Str("Before", m.PodSpecVersion).Msgf("Pod needs rotation - templates does not match") - return true, "Pod needs rotation - checksum does not match" - } - - endpoint, err := pod.GenerateMemberEndpoint(cachedStatus, apiObject, spec, group, m) - if err != nil { - log.Err(err).Msg("Error while getting pod endpoint") - return false, "" - } - - if e := m.Endpoint; e == nil { - if spec.CommunicationMethod == nil { - // TODO: Remove in 1.2.0 release to allow rotation - return false, "Pod endpoint is not set and CommunicationMethod is not set, do not recreate" - } - - return true, "Communication method has been set - ensure endpoint" - } else { - if *e != endpoint { - return true, "Pod endpoint changed" - } - } - - return false, "" -} - // clusterReadyForUpgrade returns true if the cluster is ready for the next update, that is: // - all shards are in sync // - all members are ready and fine diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index f5bc2d98a..aa5848ee6 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -300,9 +300,10 @@ func (c *testContext) GetStatus() (api.DeploymentStatus, int32) { func addAgentsToStatus(t *testing.T, status *api.DeploymentStatus, count int) { for i := 0; i < count; i++ { require.NoError(t, status.Members.Add(api.MemberStatus{ - ID: fmt.Sprintf("AGNT-%d", i), - PodName: fmt.Sprintf("agnt-depl-xxx-%d", i), - Phase: api.MemberPhaseCreated, + ID: fmt.Sprintf("AGNT-%d", i), + PodName: fmt.Sprintf("agnt-depl-xxx-%d", i), + PodSpecVersion: "random", + Phase: api.MemberPhaseCreated, Conditions: []api.Condition{ { Type: api.ConditionTypeReady, diff --git a/pkg/deployment/reconcile/plan_builder_tls.go b/pkg/deployment/reconcile/plan_builder_tls.go index 386d69036..0592fdc15 100644 --- a/pkg/deployment/reconcile/plan_builder_tls.go +++ b/pkg/deployment/reconcile/plan_builder_tls.go @@ -310,7 +310,7 @@ func createKeyfileRenewalPlanDefault(ctx context.Context, if renew, _ := keyfileRenewalRequired(lCtx, log, apiObject, spec, cachedStatus, planCtx, group, member, api.TLSRotateModeRecreate); renew { log.Info().Msg("Renewal of keyfile required - Recreate") - plan = append(plan, tlsRotateConditionPlan(group, member.ID, "Restart server after keyfile removal")...) + plan = append(plan, tlsRotateConditionAction(group, member.ID, "Restart server after keyfile removal")) } } diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go index dab65e9ae..1eee2e955 100644 --- a/pkg/deployment/reconcile/plan_builder_tls_sni.go +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -108,7 +108,7 @@ func createRotateTLSServerSNIPlan(ctx context.Context, } else if !ok { switch spec.TLS.Mode.Get() { case api.TLSRotateModeRecreate: - plan = append(plan, tlsRotateConditionPlan(group, m.ID, "SNI Secret needs update")...) + plan = append(plan, tlsRotateConditionAction(group, m.ID, "SNI Secret needs update")) case api.TLSRotateModeInPlace: plan = append(plan, api.NewAction(api.ActionTypeUpdateTLSSNI, group, m.ID, "SNI Secret needs update")) diff --git a/pkg/deployment/reconcile/plan_executor.go b/pkg/deployment/reconcile/plan_executor.go index 481521c2b..858298fc2 100644 --- a/pkg/deployment/reconcile/plan_executor.go +++ b/pkg/deployment/reconcile/plan_executor.go @@ -159,7 +159,7 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte } if abort { - return plan, true, nil + return nil, true, nil } if done { diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 6a6ef4388..f04219079 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -620,6 +620,7 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS m.PodUID = uid m.PodSpecVersion = template.PodSpecChecksum } + // Record new member phase m.Phase = newPhase m.Conditions.Remove(api.ConditionTypeReady) @@ -630,8 +631,10 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS m.Conditions.Remove(api.ConditionTypeUpgradeFailed) m.Conditions.Remove(api.ConditionTypePendingTLSRotation) m.Conditions.Remove(api.ConditionTypePendingRestart) + m.Conditions.Remove(api.ConditionTypeRestart) + m.Upgrade = false - r.log.Info().Str("DEBUG", "10101").Str("pod", m.PodName).Msgf("Updating member") + r.log.Info().Str("pod", m.PodName).Msgf("Updating member") if err := status.Members.Update(m, group); err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/rotation/check.go b/pkg/deployment/rotation/check.go new file mode 100644 index 000000000..1bb52bb29 --- /dev/null +++ b/pkg/deployment/rotation/check.go @@ -0,0 +1,119 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package rotation + +import ( + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" + "github.com/rs/zerolog" + core "k8s.io/api/core/v1" +) + +type Mode int + +const ( + EnforcedRotation Mode = iota + GracefulRotation + InPlaceRotation + SilentRotation + SkippedRotation +) + +func (m Mode) And(b Mode) Mode { + if m < b { + return m + } + + return b +} + +func IsRotationRequired(log zerolog.Logger, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, member api.MemberStatus, pod *core.Pod, specTemplate, statusTemplate *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, reason string, err error) { + // Determine if rotation is required based on plan and actions + + // Set default mode for return value + mode = SkippedRotation + + if member.Phase.IsPending() { + // Skip rotation when we are not yet created + return + } + + if spec.MemberPropagationMode.Get() == api.DeploymentMemberPropagationModeAlways && member.Conditions.IsTrue(api.ConditionTypePendingRestart) { + reason = "Restart is pending" + mode = EnforcedRotation + return + } + + // Check if pod details are propagated + if pod != nil { + if member.PodUID != pod.UID { + reason = "Pod UID does not match, this pod is not managed by Operator. Recreating" + mode = EnforcedRotation + return + } + + if _, ok := pod.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok { + reason = "Recreation enforced by annotation" + mode = EnforcedRotation + return + } + } + + if member.PodSpecVersion == "" { + reason = "Pod Spec Version is nil - recreating pod" + mode = EnforcedRotation + return + } + + if specTemplate == nil || statusTemplate == nil { + // If spec or status is nil rotation is not needed + return + } + + // Check if any of resize events are in place + if member.Conditions.IsTrue(api.ConditionTypePendingTLSRotation) { + reason = "TLS Rotation pending" + mode = EnforcedRotation + return + } + + pvc, exists := cachedStatus.PersistentVolumeClaim(member.PersistentVolumeClaimName) + if exists { + if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) { + reason = "PVC Resize pending" + mode = EnforcedRotation + return + } + } + + if statusTemplate.RotationNeeded(specTemplate) { + reason = "Pod needs rotation - templates does not match" + mode = GracefulRotation + log.Info().Str("id", member.ID).Str("Before", member.PodSpecVersion).Msgf(reason) + return + } + + return +}