From aa8a2f4011e88e6513dfea44957b858e325dcfa5 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Tue, 10 Mar 2020 06:46:43 +0000 Subject: [PATCH 1/5] Add rotate annotation --- pkg/apis/deployment/v1/annotations.go | 29 +++++++++++++++ pkg/deployment/deployment_inspector.go | 6 +--- .../reconcile/plan_builder_rotate_upgrade.go | 11 ++++++ pkg/util/k8sutil/map.go | 35 +++++++++++++++++-- 4 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 pkg/apis/deployment/v1/annotations.go diff --git a/pkg/apis/deployment/v1/annotations.go b/pkg/apis/deployment/v1/annotations.go new file mode 100644 index 000000000..e4bdb843b --- /dev/null +++ b/pkg/apis/deployment/v1/annotations.go @@ -0,0 +1,29 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package v1 + +const ( + ArangoDeploymentAnnotationPrefix = "deployment.arangodb.com" + ArangoDeploymentPodMaintenanceAnnotation = ArangoDeploymentAnnotationPrefix + "/maintenance" + ArangoDeploymentPodRotateAnnotation = ArangoDeploymentAnnotationPrefix + "/rotate" +) diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index aab2cc399..c1c59f84c 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -37,10 +37,6 @@ var ( inspectDeploymentDurationGauges = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_deployment_duration", "Amount of time taken by a single inspection of a deployment (in sec)", metrics.DeploymentName) ) -const ( - arangoDeploymentMaintenanceAnnotation = "deployment.arangodb.com/maintenance" -) - // inspectDeployment inspects the entire deployment, creates // a plan to update if needed and inspects underlying resources. // This function should be called when: @@ -74,7 +70,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval } else { // Check if maintenance annotation is set if updated != nil && updated.Annotations != nil { - if v, ok := updated.Annotations[arangoDeploymentMaintenanceAnnotation]; ok && v == "true" { + if v, ok := updated.Annotations[api.ArangoDeploymentPodMaintenanceAnnotation]; ok && v == "true" { // Disable checks if we will enter maintenance mode log.Info().Str("deployment", deploymentName).Msg("Deployment in maintenance mode") return nextInterval diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index f9254f344..d8f6a9e38 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -87,6 +87,17 @@ func createRotateOrUpgradePlan(log zerolog.Logger, apiObject k8sutil.APIObject, newPlan = createRotateMemberPlan(log, m, group, reason) } } + + if !newPlan.IsEmpty() { + // Only rotate/upgrade 1 pod at a time + continue + } + + if pod.Annotations != nil { + if _, ok := pod.Annotations[api.ArangoDeploymentPodRotateAnnotation]; ok { + createRotateMemberPlan(log, m, group, "Rotation flag present") + } + } } return nil }) diff --git a/pkg/util/k8sutil/map.go b/pkg/util/k8sutil/map.go index fd1c6ebd7..59c15c1de 100644 --- a/pkg/util/k8sutil/map.go +++ b/pkg/util/k8sutil/map.go @@ -22,11 +22,14 @@ package k8sutil -import "regexp" +import ( + v1 "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "regexp" +) const ( kubernetesAnnotationMatch = ".*kubernetes\\.io/.*" - arangoAnnotationMatch = ".*arangodb\\.com/" + arangoAnnotationMatch = ".*arangodb\\.com/.*" ) var ( @@ -50,6 +53,29 @@ func init() { arangoAnnotationRegex = r } +func isFilteredBlockedAnnotation(key string) bool { + switch key { + case v1.ArangoDeploymentPodRotateAnnotation: + return true + default: + return false + } +} + +func filterBlockedAnnotations(m map[string]string) map[string]string { + n := map[string]string{} + + for key, value := range m { + if isFilteredBlockedAnnotation(key) { + continue + } + + n[key] = value + } + + return n +} + // MergeAnnotations into one annotations map func MergeAnnotations(annotations ...map[string]string) map[string]string { ret := map[string]string{} @@ -114,7 +140,12 @@ func filterActualAnnotations(actual, expected map[string]string) map[string]stri // CompareAnnotations will compare annotations, but will ignore secured annotations which are present in // actual but not specified in expected map +// It will also filter out blocked annotations func CompareAnnotations(actual, expected map[string]string) bool { + return compareAnnotations(filterBlockedAnnotations(actual), filterBlockedAnnotations(expected)) +} + +func compareAnnotations(actual, expected map[string]string) bool { actualFiltered := filterActualAnnotations(actual, expected) if actualFiltered == nil && expected == nil { From 59d9f163241e23985ec76f801a6de28454980cdb Mon Sep 17 00:00:00 2001 From: ajanikow Date: Tue, 10 Mar 2020 07:03:05 +0000 Subject: [PATCH 2/5] Fix import cycle --- pkg/apis/deployment/{v1 => }/annotations.go | 2 +- pkg/util/k8sutil/map.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename pkg/apis/deployment/{v1 => }/annotations.go (98%) diff --git a/pkg/apis/deployment/v1/annotations.go b/pkg/apis/deployment/annotations.go similarity index 98% rename from pkg/apis/deployment/v1/annotations.go rename to pkg/apis/deployment/annotations.go index e4bdb843b..f47e26353 100644 --- a/pkg/apis/deployment/v1/annotations.go +++ b/pkg/apis/deployment/annotations.go @@ -20,7 +20,7 @@ // Author Adam Janikowski // -package v1 +package deployment const ( ArangoDeploymentAnnotationPrefix = "deployment.arangodb.com" diff --git a/pkg/util/k8sutil/map.go b/pkg/util/k8sutil/map.go index 59c15c1de..0bb574c17 100644 --- a/pkg/util/k8sutil/map.go +++ b/pkg/util/k8sutil/map.go @@ -23,7 +23,7 @@ package k8sutil import ( - v1 "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" "regexp" ) @@ -55,7 +55,7 @@ func init() { func isFilteredBlockedAnnotation(key string) bool { switch key { - case v1.ArangoDeploymentPodRotateAnnotation: + case deployment.ArangoDeploymentPodRotateAnnotation: return true default: return false From 709b3a0ea3497a0d92b722054bba8b296fa0b7ca Mon Sep 17 00:00:00 2001 From: ajanikow Date: Tue, 10 Mar 2020 08:48:25 +0000 Subject: [PATCH 3/5] Move flags --- pkg/deployment/deployment_inspector.go | 3 ++- pkg/deployment/reconcile/plan_builder_rotate_upgrade.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index c1c59f84c..5ce425cb2 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -24,6 +24,7 @@ package deployment import ( "context" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" "time" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -70,7 +71,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval } else { // Check if maintenance annotation is set if updated != nil && updated.Annotations != nil { - if v, ok := updated.Annotations[api.ArangoDeploymentPodMaintenanceAnnotation]; ok && v == "true" { + if v, ok := updated.Annotations[deployment.ArangoDeploymentPodMaintenanceAnnotation]; ok && v == "true" { // Disable checks if we will enter maintenance mode log.Info().Str("deployment", deploymentName).Msg("Deployment in maintenance mode") return nextInterval diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index d8f6a9e38..ce3e8b62e 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -23,6 +23,8 @@ package reconcile import ( + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "reflect" "strings" @@ -94,8 +96,8 @@ func createRotateOrUpgradePlan(log zerolog.Logger, apiObject k8sutil.APIObject, } if pod.Annotations != nil { - if _, ok := pod.Annotations[api.ArangoDeploymentPodRotateAnnotation]; ok { - createRotateMemberPlan(log, m, group, "Rotation flag present") + if _, ok := pod.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok { + newPlan = createRotateMemberPlan(log, m, group, "Rotation flag present") } } } From 8df75bfa16da8a733aa1ae28bf51fb794891a228 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Tue, 10 Mar 2020 09:29:23 +0000 Subject: [PATCH 4/5] Adjust linter --- pkg/deployment/deployment_inspector.go | 3 ++- pkg/deployment/reconcile/plan_builder_rotate_upgrade.go | 5 +++-- pkg/util/k8sutil/map.go | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 5ce425cb2..98686ba15 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -24,9 +24,10 @@ package deployment import ( "context" - "github.com/arangodb/kube-arangodb/pkg/apis/deployment" "time" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/metrics" "github.com/arangodb/kube-arangodb/pkg/util" diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index ce3e8b62e..42b555ffa 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -23,11 +23,12 @@ package reconcile import ( - "github.com/arangodb/kube-arangodb/pkg/apis/deployment" - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "reflect" "strings" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "github.com/arangodb/go-driver" diff --git a/pkg/util/k8sutil/map.go b/pkg/util/k8sutil/map.go index 0bb574c17..010a672ed 100644 --- a/pkg/util/k8sutil/map.go +++ b/pkg/util/k8sutil/map.go @@ -23,8 +23,9 @@ package k8sutil import ( - "github.com/arangodb/kube-arangodb/pkg/apis/deployment" "regexp" + + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" ) const ( From 81e9a00a4436612bc1665a33b91ad1e514290e31 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Tue, 10 Mar 2020 13:56:32 +0000 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 1 + docs/design/README.md | 1 + docs/design/rotating.md | 13 +++++++++++++ 3 files changed, 15 insertions(+) create mode 100644 docs/design/rotating.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 657431e04..b17eda5e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Change Log ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) +- Added annotation to rotate ArangoDeployment in secure way ## [1.0.0](https://github.com/arangodb/kube-arangodb/tree/1.0.0) (2020-03-03) - Removal of v1alpha support for ArangoDeployment, ArangoDeploymentReplication, ArangoBackup diff --git a/docs/design/README.md b/docs/design/README.md index 61fb869bb..ee091d638 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -8,4 +8,5 @@ - [Scaling](./scaling.md) - [Status](./status.md) - [Upgrading](./upgrading.md) +- [Rotating Pods](./rotating.md) - [Maintenance](./maintenance.md) \ No newline at end of file diff --git a/docs/design/rotating.md b/docs/design/rotating.md new file mode 100644 index 000000000..2cd7a1591 --- /dev/null +++ b/docs/design/rotating.md @@ -0,0 +1,13 @@ +# Rotation + +## ArangoDeployment + +Rotation of ArangoDeployment Pods can be triggered by Pod deletion or by annotation (safe way). + +Using annotation Pods gonna be rotated one-by-one which will keep cluster alive. + +Key: `deployment.arangodb.com/rotation` +Value: `true` + +To rotate ArangoDeployment Pod kubectl command can be used: +`kubectl annotate pod arango-pod deployment.arangodb.com/rotation=true` \ No newline at end of file