From 7f1790487312fcc4a54e0b0d1d660dfcd8bfebc6 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Fri, 19 Feb 2021 10:08:41 +0000 Subject: [PATCH 1/2] Fix Cache and DBServer member removal --- CHANGELOG.md | 2 ++ pkg/deployment/context_impl.go | 13 ++++++++++- pkg/deployment/deployment.go | 1 + pkg/deployment/deployment_inspector.go | 3 +++ .../reconcile/plan_builder_cluster.go | 12 ++++++++++ pkg/deployment/resources/context.go | 5 +++++ .../resources/inspector/inspector.go | 3 +++ pkg/deployment/resources/inspector/secrets.go | 22 +++++++++++++++++++ pkg/util/k8sutil/secrets.go | 7 +++++- 9 files changed, 66 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 548d4e866..42c9200e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) - Fix AKS Volume Resize mode +- Use cached status in member client creation +- Remove failed DBServers ## [1.1.4](https://github.com/arangodb/kube-arangodb/tree/1.1.4) (2021-02-15) - Add support for spec.ClusterDomain to be able to use FQDN in ArangoDB cluster communication diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index 5646b7d3c..27cbd8432 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -263,7 +263,10 @@ func (d *Deployment) getAuth() (driver.Authentication, error) { return nil, nil } - secrets := d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace()) + var secrets k8sutil.SecretReadInterface = d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace()) + if currentState := d.currentState; currentState != nil { + secrets = currentState.SecretReadInterface() + } var secret string if i := d.apiObject.Status.CurrentImage; i == nil || !features.JWTRotation().Supported(i.ArangoDBVersion, i.Enterprise) { @@ -589,3 +592,11 @@ func (d *Deployment) GetOwnedPods() ([]v1.Pod, error) { return podList, nil } + +func (d *Deployment) GetCachedStatus() inspector.Inspector { + return d.currentState +} + +func (d *Deployment) SetCachedStatus(i inspector.Inspector) { + d.currentState = i +} diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 4b33eee94..9cdedf26e 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -121,6 +121,7 @@ type Deployment struct { inspectCRDTrigger trigger.Trigger updateDeploymentTrigger trigger.Trigger clientCache deploymentClient.Cache + currentState inspector.Inspector recentInspectionErrors int clusterScalingIntegration *clusterScalingIntegration reconciler *reconcile.Reconciler diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 614f2bf8d..e1a7c1157 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -130,6 +130,9 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterval util.Interval, cachedStatus inspector.Inspector) (nextInterval util.Interval, inspectError error) { t := time.Now() + d.SetCachedStatus(cachedStatus) + defer d.SetCachedStatus(nil) + defer func() { d.deps.Log.Info().Msgf("Reconciliation loop took %s", time.Since(t)) }() diff --git a/pkg/deployment/reconcile/plan_builder_cluster.go b/pkg/deployment/reconcile/plan_builder_cluster.go index bc6cd0bba..291ae572d 100644 --- a/pkg/deployment/reconcile/plan_builder_cluster.go +++ b/pkg/deployment/reconcile/plan_builder_cluster.go @@ -88,6 +88,18 @@ func createClusterOperationPlan(ctx context.Context, api.NewAction(api.ActionTypeClusterMemberCleanup, api.ServerGroupCoordinators, string(id)), } } + case driver.ServerRoleDBServer: + if member.Status != driver.ServerStatusFailed { + continue + } + + if !member.CanBeDeleted { + continue + } + + return api.Plan{ + api.NewAction(api.ActionTypeClusterMemberCleanup, api.ServerGroupDBServers, string(id)), + } } } diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index 68a53eaaa..5cc98dbd0 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -25,6 +25,8 @@ package resources import ( "context" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" + "github.com/arangodb/kube-arangodb/pkg/operator/scope" monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" @@ -103,4 +105,7 @@ type Context interface { // GetBackup receives information about a backup resource GetBackup(backup string) (*backupApi.ArangoBackup, error) GetScope() scope.Scope + + GetCachedStatus() inspector.Inspector + SetCachedStatus(i inspector.Inspector) } diff --git a/pkg/deployment/resources/inspector/inspector.go b/pkg/deployment/resources/inspector/inspector.go index 37d96a2ed..f2944c5a0 100644 --- a/pkg/deployment/resources/inspector/inspector.go +++ b/pkg/deployment/resources/inspector/inspector.go @@ -25,6 +25,8 @@ package inspector import ( "sync" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" @@ -102,6 +104,7 @@ type Inspector interface { Secret(name string) (*core.Secret, bool) IterateSecrets(action SecretAction, filters ...SecretFilter) error + SecretReadInterface() k8sutil.SecretReadInterface PersistentVolumeClaim(name string) (*core.PersistentVolumeClaim, bool) IteratePersistentVolumeClaims(action PersistentVolumeClaimAction, filters ...PersistentVolumeClaimFilter) error diff --git a/pkg/deployment/resources/inspector/secrets.go b/pkg/deployment/resources/inspector/secrets.go index 5fd2ac981..e1ddccee0 100644 --- a/pkg/deployment/resources/inspector/secrets.go +++ b/pkg/deployment/resources/inspector/secrets.go @@ -24,8 +24,11 @@ package inspector import ( "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" core "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" ) @@ -66,6 +69,25 @@ func (i *inspector) Secret(name string) (*core.Secret, bool) { return secret, true } +func (i *inspector) SecretReadInterface() k8sutil.SecretReadInterface { + return &secretReadInterface{i: i} +} + +type secretReadInterface struct { + i *inspector +} + +func (s secretReadInterface) Get(name string, options meta.GetOptions) (*core.Secret, error) { + if s, ok := s.i.Secret(name); !ok { + return nil, apiErrors.NewNotFound(schema.GroupResource{ + Group: core.GroupName, + Resource: "Secret", + }, name) + } else { + return s, nil + } +} + func secretsToMap(k kubernetes.Interface, namespace string) (map[string]*core.Secret, error) { secrets, err := getSecrets(k, namespace, "") if err != nil { diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 769628155..d6c658357 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -32,11 +32,16 @@ import ( jg "github.com/dgrijalva/jwt-go" ) +// SecretReadInterface has methods to work with Secret resources with ReadOnly mode. +type SecretReadInterface interface { + Get(name string, options meta.GetOptions) (*core.Secret, error) +} + // SecretInterface has methods to work with Secret resources. type SecretInterface interface { + SecretReadInterface Create(*core.Secret) (*core.Secret, error) Update(*core.Secret) (*core.Secret, error) - Get(name string, options meta.GetOptions) (*core.Secret, error) Delete(name string, options *meta.DeleteOptions) error Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*core.Secret, error) } From 3c1f62c8375864734ef4dd3656fad2a88f405191 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Fri, 19 Feb 2021 10:48:37 +0000 Subject: [PATCH 2/2] Fix cycle --- pkg/deployment/context_impl.go | 2 +- pkg/deployment/resources/inspector/inspector.go | 9 +++++++-- pkg/deployment/resources/inspector/secrets.go | 3 +-- pkg/util/k8sutil/secrets.go | 7 +------ 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index 27cbd8432..cd21f2123 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -263,7 +263,7 @@ func (d *Deployment) getAuth() (driver.Authentication, error) { return nil, nil } - var secrets k8sutil.SecretReadInterface = d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace()) + var secrets inspector.SecretReadInterface = d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace()) if currentState := d.currentState; currentState != nil { secrets = currentState.SecretReadInterface() } diff --git a/pkg/deployment/resources/inspector/inspector.go b/pkg/deployment/resources/inspector/inspector.go index f2944c5a0..7e9601a53 100644 --- a/pkg/deployment/resources/inspector/inspector.go +++ b/pkg/deployment/resources/inspector/inspector.go @@ -25,7 +25,7 @@ package inspector import ( "sync" - "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" @@ -35,6 +35,11 @@ import ( "k8s.io/client-go/kubernetes" ) +// SecretReadInterface has methods to work with Secret resources with ReadOnly mode. +type SecretReadInterface interface { + Get(name string, options meta.GetOptions) (*core.Secret, error) +} + func NewInspector(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, namespace string) (Inspector, error) { pods, err := podsToMap(k, namespace) if err != nil { @@ -104,7 +109,7 @@ type Inspector interface { Secret(name string) (*core.Secret, bool) IterateSecrets(action SecretAction, filters ...SecretFilter) error - SecretReadInterface() k8sutil.SecretReadInterface + SecretReadInterface() SecretReadInterface PersistentVolumeClaim(name string) (*core.PersistentVolumeClaim, bool) IteratePersistentVolumeClaims(action PersistentVolumeClaimAction, filters ...PersistentVolumeClaimFilter) error diff --git a/pkg/deployment/resources/inspector/secrets.go b/pkg/deployment/resources/inspector/secrets.go index e1ddccee0..e8f8c2c6e 100644 --- a/pkg/deployment/resources/inspector/secrets.go +++ b/pkg/deployment/resources/inspector/secrets.go @@ -24,7 +24,6 @@ package inspector import ( "github.com/arangodb/kube-arangodb/pkg/util/errors" - "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" core "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -69,7 +68,7 @@ func (i *inspector) Secret(name string) (*core.Secret, bool) { return secret, true } -func (i *inspector) SecretReadInterface() k8sutil.SecretReadInterface { +func (i *inspector) SecretReadInterface() SecretReadInterface { return &secretReadInterface{i: i} } diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index d6c658357..930bee96b 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -32,14 +32,9 @@ import ( jg "github.com/dgrijalva/jwt-go" ) -// SecretReadInterface has methods to work with Secret resources with ReadOnly mode. -type SecretReadInterface interface { - Get(name string, options meta.GetOptions) (*core.Secret, error) -} - // SecretInterface has methods to work with Secret resources. type SecretInterface interface { - SecretReadInterface + Get(name string, options meta.GetOptions) (*core.Secret, error) Create(*core.Secret) (*core.Secret, error) Update(*core.Secret) (*core.Secret, error) Delete(name string, options *meta.DeleteOptions) error