[Feature] Disaster recovery

CHANGELOG.md

@@ -10,6 +10,8 @@
 - Add Label and Envs Pod customization
 - Improved JWT Rotation
 - Allow to customize Security Context in pods
+- Remove dead Coordinators in Cluster mode
+- Add AutoRecovery flag to recover cluster in case of deadlock
 
 ## [1.0.3](https://github.com/arangodb/kube-arangodb/tree/1.0.3) (2020-05-25)
 - Prevent deletion of not known PVC's

pkg/apis/deployment/v1/deployment_spec.go

@@ -94,6 +94,8 @@ type DeploymentSpec struct {
 
 	Chaos ChaosSpec `json:"chaos"`
 
+	Recovery *ArangoDeploymentRecoverySpec `json:"recovery,omitempty"`
+
 	Bootstrap BootstrapSpec `json:"bootstrap,omitempty"`
 }
 

pkg/apis/deployment/v1/plan.go

@@ -113,6 +113,8 @@ const (
 	ActionTypeJWTRefresh ActionType = "JWTRefresh"
 	// ActionTypeJWTPropagated change propagated flag
 	ActionTypeJWTPropagated ActionType = "JWTPropagated"
+	// ActionTypeClusterMemberCleanup removes member from cluster
+	ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup"
 )
 
 const (

pkg/apis/deployment/v1/recovery_spec.go

@@ -0,0 +1,41 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package v1
+
+import "github.com/arangodb/kube-arangodb/pkg/util"
+
+type ArangoDeploymentRecoverySpec struct {
+	AutoRecover *bool `json:"autoRecover"`
+}
+
+func (a *ArangoDeploymentRecoverySpec) Get() ArangoDeploymentRecoverySpec {
+	if a != nil {
+		return *a
+	}
+
+	return ArangoDeploymentRecoverySpec{}
+}
+
+func (a ArangoDeploymentRecoverySpec) GetAutoRecover() bool {
+	return util.BoolOrDefault(a.AutoRecover, false)
+}

pkg/apis/deployment/v1/server_group_spec.go

@@ -103,7 +103,7 @@ type ServerGroupSpecSecurityContext struct {
 
 	AllowPrivilegeEscalation *bool  `json:"allowPrivilegeEscalation,omitempty"`
 	Privileged               *bool  `json:"privileged,omitempty"`
-	ReadOnlyRootFilesystem   *bool  `json:"readOnlyFileSystem,omitempty"`
+	ReadOnlyRootFilesystem   *bool  `json:"readOnlyRootFilesystem,omitempty"`
 	RunAsNonRoot             *bool  `json:"runAsNonRoot,omitempty"`
 	RunAsUser                *int64 `json:"runAsUser,omitempty"`
 	RunAsGroup               *int64 `json:"runAsGroup,omitempty"`

pkg/deployment/reconcile/action_cluster_member_cleanup.go

@@ -0,0 +1,95 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/go-driver"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeClusterMemberCleanup, newClusterMemberCleanupAction)
+}
+
+// newClusterMemberCleanupAction creates a new Action that implements the given
+// planned ClusterMemberCleanup action.
+func newClusterMemberCleanupAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionClusterMemberCleanup{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, addMemberTimeout)
+
+	return a
+}
+
+// actionClusterMemberCleanup implements an ClusterMemberCleanup.
+type actionClusterMemberCleanup struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	// actionEmptyCheckProgress implement check progress with empty implementation
+	actionEmptyCheckProgress
+}
+
+// Start performs the start of the action.
+// Returns true if the action is completely finished, false in case
+// the start time needs to be recorded and a ready condition needs to be checked.
+func (a *actionClusterMemberCleanup) Start(ctx context.Context) (bool, error) {
+	if err := a.start(ctx); err != nil {
+		a.log.Warn().Err(err).Msgf("Unable to clean cluster member")
+	}
+
+	return true, nil
+}
+
+func (a *actionClusterMemberCleanup) start(ctx context.Context) error {
+	id := driver.ServerID(a.MemberID())
+
+	c, err := a.actionCtx.GetDatabaseClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	cluster, err := c.Cluster(ctx)
+	if err != nil {
+		return err
+	}
+
+	health, err := cluster.Health(ctx)
+	if err != nil {
+		return err
+	}
+
+	if _, ok := health.Health[id]; !ok {
+		return nil
+	}
+
+	if err := cluster.RemoveServer(ctx, id); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder.go

@@ -245,11 +245,6 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = pb.Apply(createKeyfileRenewalPlan)
 	}
 
-	// Check for the need to rotate TLS certificate of a members
-	//if plan.IsEmpty() {
-	//	plan = pb.Apply(createRotateTLSServerCertificatePlan)
-	//}
-
 	// Check for changes storage classes or requirements
 	if plan.IsEmpty() {
 		plan = pb.Apply(createRotateServerStoragePlan)
@@ -271,6 +266,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = pb.Apply(createCACleanPlan)
 	}
 
+	if plan.IsEmpty() {
+		plan = pb.Apply(createClusterOperationPlan)
+	}
+
 	// Return plan
 	return plan, true
 }

pkg/deployment/reconcile/plan_builder_cluster.go

@@ -0,0 +1,95 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+	"time"
+
+	"github.com/arangodb/go-driver"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/rs/zerolog"
+)
+
+const coordinatorHealthFailedTimeout time.Duration = time.Minute
+
+func createClusterOperationPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+
+	if spec.GetMode() != api.DeploymentModeCluster {
+		return nil
+	}
+
+	c, err := context.GetDatabaseClient(ctx)
+	if err != nil {
+		return nil
+	}
+
+	cluster, err := c.Cluster(ctx)
+	if err != nil {
+		log.Warn().Err(err).Msgf("Unable to get Cluster client")
+		return nil
+	}
+
+	health, err := cluster.Health(ctx)
+	if err != nil {
+		log.Warn().Err(err).Msgf("Unable to get Cluster health")
+		return nil
+	}
+
+	membersHealth := health.Health
+
+	status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
+		for _, m := range list {
+			delete(membersHealth, driver.ServerID(m.ID))
+		}
+
+		return nil
+	})
+
+	if len(membersHealth) == 0 {
+		return nil
+	}
+
+	for id, member := range membersHealth {
+		switch member.Role {
+		case driver.ServerRoleCoordinator:
+			if member.Status != driver.ServerStatusFailed {
+				continue
+			}
+
+			if member.LastHeartbeatAcked.Add(coordinatorHealthFailedTimeout).Before(time.Now()) {
+				return api.Plan{
+					api.NewAction(api.ActionTypeClusterMemberCleanup, api.ServerGroupCoordinators, string(id)),
+				}
+			}
+		}
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder_context.go

@@ -58,6 +58,9 @@ type PlanBuilderContext interface {
 	RenderPodForMember(cachedStatus inspector.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error)
 	// SelectImage select currently used image by pod
 	SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool)
+	// GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server),
+	// creating one if needed.
+	GetDatabaseClient(ctx context.Context) (driver.Client, error)
 	// GetServerClient returns a cached client for a specific server.
 	GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error)
 	// SecretsInterface return secret interface

pkg/deployment/reconcile/plan_builder_test.go

@@ -28,6 +28,8 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/pkg/errors"
+
 	policy "k8s.io/api/policy/v1beta1"
 
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
@@ -117,7 +119,7 @@ func (c *testContext) UpdateMember(member api.MemberStatus) error {
 }
 
 func (c *testContext) GetDatabaseClient(ctx context.Context) (driver.Client, error) {
-	panic("implement me")
+	return nil, errors.Errorf("Client Not Found")
 }
 
 func (c *testContext) GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) {