Feature/tg 165 kill pod forcefully

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Do not check License V2 on Community images
 - Add status.members.<group>.
 - Define MemberReplacementRequired condition
+- Remove pod immediately when annotation is turned on
 
 ## [1.2.7](https://github.com/arangodb/kube-arangodb/tree/1.2.7) (2022-01-17)
 - Add Plan BackOff functionality

pkg/apis/deployment/annotations.go

@@ -25,5 +25,6 @@ const (
 	ArangoDeploymentPodMaintenanceAnnotation = ArangoDeploymentAnnotationPrefix + "/maintenance"
 	ArangoDeploymentPodRotateAnnotation      = ArangoDeploymentAnnotationPrefix + "/rotate"
 	ArangoDeploymentPodReplaceAnnotation     = ArangoDeploymentAnnotationPrefix + "/replace"
+	ArangoDeploymentPodDeleteNow             = ArangoDeploymentAnnotationPrefix + "/delete_now"
 	ArangoDeploymentPlanCleanAnnotation      = "plan." + ArangoDeploymentAnnotationPrefix + "/clean"
 )

pkg/apis/deployment/v1/conditions.go

@@ -59,7 +59,7 @@ const (
 
 	// ConditionTypeTerminating indicates that the member is terminating but not yet terminated.
 	ConditionTypeTerminating ConditionType = "Terminating"
-	// ConditionTypeTerminating indicates that the deployment is up to date.
+	// ConditionTypeUpToDate indicates that the deployment is up to date.
 	ConditionTypeUpToDate ConditionType = "UpToDate"
 	// ConditionTypeMarkedToRemove indicates that the member is marked to be removed.
 	ConditionTypeMarkedToRemove ConditionType = "MarkedToRemove"

pkg/apis/deployment/v1/plan.go

@@ -77,7 +77,7 @@ const (
 	ActionTypeRotateMember ActionType = "RotateMember"
 	// ActionTypeRotateStartMember causes a member to be shutdown and have it's pod removed. Do not wait to pod recover.
 	ActionTypeRotateStartMember ActionType = "RotateStartMember"
-	// ActionTypeRotateMember causes a member to be restored.
+	// ActionTypeRotateStopMember causes a member to be restored.
 	ActionTypeRotateStopMember ActionType = "RotateStopMember"
 	// ActionTypeUpgradeMember causes a member to be shutdown and have it's pod removed, restarted with AutoUpgrade option, waited until termination and the restarted again.
 	ActionTypeUpgradeMember ActionType = "UpgradeMember"
@@ -105,7 +105,7 @@ const (
 	ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI"
 	// ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action.
 	ActionTypeSetCurrentImage ActionType = "SetCurrentImage"
-	// ActionTypeSetCurrentImage replace image of member to current one.
+	// ActionTypeSetMemberCurrentImage replace image of member to current one.
 	ActionTypeSetMemberCurrentImage ActionType = "SetMemberCurrentImage"
 	// ActionTypeDisableClusterScaling turns off scaling DBservers and coordinators
 	ActionTypeDisableClusterScaling ActionType = "ScalingDisabled"
@@ -115,7 +115,7 @@ const (
 	ActionTypePVCResize ActionType = "PVCResize"
 	// ActionTypePVCResized waits for PVC to resize for defined time
 	ActionTypePVCResized ActionType = "PVCResized"
-	// UpToDateUpdateResized define up to date annotation in spec
+	// UpToDateUpdate define up to date annotation in spec
 	UpToDateUpdate ActionType = "UpToDateUpdate"
 	// ActionTypeBackupRestore restore plan
 	ActionTypeBackupRestore ActionType = "BackupRestore"

pkg/apis/deployment/v2alpha1/conditions.go

@@ -59,7 +59,7 @@ const (
 
 	// ConditionTypeTerminating indicates that the member is terminating but not yet terminated.
 	ConditionTypeTerminating ConditionType = "Terminating"
-	// ConditionTypeTerminating indicates that the deployment is up to date.
+	// ConditionTypeUpToDate indicates that the deployment is up to date.
 	ConditionTypeUpToDate ConditionType = "UpToDate"
 	// ConditionTypeMarkedToRemove indicates that the member is marked to be removed.
 	ConditionTypeMarkedToRemove ConditionType = "MarkedToRemove"

pkg/apis/deployment/v2alpha1/plan.go

@@ -77,7 +77,7 @@ const (
 	ActionTypeRotateMember ActionType = "RotateMember"
 	// ActionTypeRotateStartMember causes a member to be shutdown and have it's pod removed. Do not wait to pod recover.
 	ActionTypeRotateStartMember ActionType = "RotateStartMember"
-	// ActionTypeRotateMember causes a member to be restored.
+	// ActionTypeRotateStopMember causes a member to be restored.
 	ActionTypeRotateStopMember ActionType = "RotateStopMember"
 	// ActionTypeUpgradeMember causes a member to be shutdown and have it's pod removed, restarted with AutoUpgrade option, waited until termination and the restarted again.
 	ActionTypeUpgradeMember ActionType = "UpgradeMember"
@@ -105,7 +105,7 @@ const (
 	ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI"
 	// ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action.
 	ActionTypeSetCurrentImage ActionType = "SetCurrentImage"
-	// ActionTypeSetCurrentImage replace image of member to current one.
+	// ActionTypeSetMemberCurrentImage replace image of member to current one.
 	ActionTypeSetMemberCurrentImage ActionType = "SetMemberCurrentImage"
 	// ActionTypeDisableClusterScaling turns off scaling DBservers and coordinators
 	ActionTypeDisableClusterScaling ActionType = "ScalingDisabled"
@@ -115,7 +115,7 @@ const (
 	ActionTypePVCResize ActionType = "PVCResize"
 	// ActionTypePVCResized waits for PVC to resize for defined time
 	ActionTypePVCResized ActionType = "PVCResized"
-	// UpToDateUpdateResized define up to date annotation in spec
+	// UpToDateUpdate define up to date annotation in spec
 	UpToDateUpdate ActionType = "UpToDateUpdate"
 	// ActionTypeBackupRestore restore plan
 	ActionTypeBackupRestore ActionType = "BackupRestore"

pkg/deployment/chaos/context.go

@@ -23,9 +23,9 @@ package chaos
 import (
 	"context"
 
-	v1 "k8s.io/api/core/v1"
-
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	v1 "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // Context provides methods to the chaos package.
@@ -34,7 +34,7 @@ type Context interface {
 	GetSpec() api.DeploymentSpec
 	// DeletePod deletes a pod with given name in the namespace
 	// of the deployment. If the pod does not exist, the error is ignored.
-	DeletePod(ctx context.Context, podName string) error
+	DeletePod(ctx context.Context, podName string, options meta.DeleteOptions) error
 	// GetOwnedPods returns a list of all pods owned by the deployment.
 	GetOwnedPods(ctx context.Context) ([]v1.Pod, error)
 }

pkg/deployment/chaos/monkey.go

@@ -25,10 +25,11 @@ import (
 	"math/rand"
 	"time"
 
-	"github.com/arangodb/kube-arangodb/pkg/util/errors"
-
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
 // Monkey is the service that introduces chaos in the deployment
@@ -87,7 +88,7 @@ func (m Monkey) killRandomPod(ctx context.Context) error {
 	}
 	p := pods[rand.Intn(len(pods))]
 	m.log.Info().Str("pod-name", p.GetName()).Msg("Killing pod")
-	if err := m.context.DeletePod(ctx, p.GetName()); err != nil {
+	if err := m.context.DeletePod(ctx, p.GetName(), meta.DeleteOptions{}); err != nil {
 		return errors.WithStack(err)
 	}
 	return nil

pkg/deployment/context_impl.go

@@ -389,10 +389,10 @@ func (d *Deployment) GetPod(ctx context.Context, podName string) (*core.Pod, err
 
 // DeletePod deletes a pod with given name in the namespace
 // of the deployment. If the pod does not exist, the error is ignored.
-func (d *Deployment) DeletePod(ctx context.Context, podName string) error {
+func (d *Deployment) DeletePod(ctx context.Context, podName string, options meta.DeleteOptions) error {
 	log := d.deps.Log
 	err := globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error {
-		return d.PodsModInterface().Delete(ctxChild, podName, meta.DeleteOptions{})
+		return d.PodsModInterface().Delete(ctxChild, podName, options)
 	})
 	if err != nil && !k8sutil.IsNotFound(err) {
 		log.Debug().Err(err).Str("pod", podName).Msg("Failed to remove pod")

pkg/deployment/reconcile/action.go

@@ -125,3 +125,20 @@ func getActionFactory(t api.ActionType) (actionFactory, bool) {
 	f, ok := actions[t]
 	return f, ok
 }
+
+type actionSuccess struct{}
+
+// NewActionSuccess returns action which always returns success.
+func NewActionSuccess() ActionCore {
+	return actionSuccess{}
+}
+
+// Start always returns true.
+func (actionSuccess) Start(_ context.Context) (bool, error) {
+	return true, nil
+}
+
+// CheckProgress always returns true.
+func (actionSuccess) CheckProgress(_ context.Context) (bool, bool, error) {
+	return true, false, nil
+}

pkg/deployment/reconcile/action_context.go

@@ -23,8 +23,22 @@ package reconcile
 import (
 	"context"
 
-	agencyCache "github.com/arangodb/kube-arangodb/pkg/deployment/agency"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/arangodb/arangosync-client/client"
+	"github.com/arangodb/go-driver"
+	"github.com/arangodb/go-driver/agency"
+
+	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	agencyCache "github.com/arangodb/kube-arangodb/pkg/deployment/agency"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod"
@@ -33,25 +47,6 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/service"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/serviceaccount"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/servicemonitor"
-
-	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
-
-	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
-
-	"github.com/arangodb/kube-arangodb/pkg/util/errors"
-
-	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
-
-	"github.com/arangodb/go-driver/agency"
-	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
-	core "k8s.io/api/core/v1"
-
-	"github.com/arangodb/arangosync-client/client"
-	driver "github.com/arangodb/go-driver"
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
-
-	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 )
 
 // ActionContext provides methods to the Action implementations
@@ -103,7 +98,7 @@ type ActionContext interface {
 	GetPod(ctx context.Context, podName string) (*core.Pod, error)
 	// DeletePod deletes a pod with given name in the namespace
 	// of the deployment. If the pod does not exist, the error is ignored.
-	DeletePod(ctx context.Context, podName string) error
+	DeletePod(ctx context.Context, podName string, options meta.DeleteOptions) error
 	// DeletePvc deletes a persistent volume claim with given name in the namespace
 	// of the deployment. If the pvc does not exist, the error is ignored.
 	DeletePvc(ctx context.Context, pvcName string) error
@@ -427,8 +422,8 @@ func (ac *actionContext) GetPod(ctx context.Context, podName string) (*core.Pod,
 
 // DeletePod deletes a pod with given name in the namespace
 // of the deployment. If the pod does not exist, the error is ignored.
-func (ac *actionContext) DeletePod(ctx context.Context, podName string) error {
-	if err := ac.context.DeletePod(ctx, podName); err != nil {
+func (ac *actionContext) DeletePod(ctx context.Context, podName string, options meta.DeleteOptions) error {
+	if err := ac.context.DeletePod(ctx, podName, options); err != nil {
 		return errors.WithStack(err)
 	}
 	return nil

pkg/deployment/reconcile/action_kill_member_pod.go

@@ -23,12 +23,13 @@ package reconcile
 import (
 	"context"
 
+	"github.com/rs/zerolog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/handlers/utils"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
-
-	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/rs/zerolog"
 )
 
 func init() {
@@ -66,7 +67,7 @@ func (a *actionKillMemberPod) Start(ctx context.Context) (bool, error) {
 		return true, nil
 	}
 
-	if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil {
+	if err := a.actionCtx.DeletePod(ctx, m.PodName, meta.DeleteOptions{}); err != nil {
 		log.Error().Err(err).Msg("Unable to kill pod")
 		return true, nil
 	}

pkg/deployment/reconcile/action_remove_member.go

@@ -23,17 +23,15 @@ package reconcile
 import (
 	"context"
 
-	"github.com/arangodb/kube-arangodb/pkg/util/globals"
-
-	apiErrors "k8s.io/apimachinery/pkg/api/errors"
-
-	"github.com/arangodb/kube-arangodb/pkg/util/errors"
-
+	"github.com/arangodb/go-driver"
 	"github.com/rs/zerolog"
+	apiErrors "k8s.io/apimachinery/pkg/api/errors"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	driver "github.com/arangodb/go-driver"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/globals"
 )
 
 func init() {
@@ -108,7 +106,7 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) {
 	}
 	if m.PodName != "" {
 		// Remove the pod (if any)
-		if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil {
+		if err := a.actionCtx.DeletePod(ctx, m.PodName, meta.DeleteOptions{}); err != nil {
 			if !apiErrors.IsNotFound(err) {
 				return false, errors.WithStack(err)
 			}

pkg/deployment/reconcile/action_rotate_member.go

@@ -23,12 +23,12 @@ package reconcile
 import (
 	"context"
 
-	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
-
-	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/rs/zerolog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/rs/zerolog"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 func init() {
@@ -55,13 +55,12 @@ type actionRotateMember struct {
 // Returns true if the action is completely finished, false in case
 // the start time needs to be recorded and a ready condition needs to be checked.
 func (a *actionRotateMember) Start(ctx context.Context) (bool, error) {
-	log := a.log
-	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	shutdown, m, ok := getShutdownHelper(&a.action, a.actionCtx, a.log)
 	if !ok {
-		log.Error().Msg("No such member")
+		return true, nil
 	}
 
-	if ready, err := getShutdownHelper(&a.action, a.actionCtx, a.log).Start(ctx); err != nil {
+	if ready, err := shutdown.Start(ctx); err != nil {
 		return false, err
 	} else if ready {
 		return true, nil
@@ -81,20 +80,19 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) {
 func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, error) {
 	// Check that pod is removed
 	log := a.log
-	m, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
-	if !found {
-		log.Error().Msg("No such member")
+	shutdown, m, ok := getShutdownHelper(&a.action, a.actionCtx, a.log)
+	if !ok {
 		return true, false, nil
 	}
 
-	if ready, abort, err := getShutdownHelper(&a.action, a.actionCtx, a.log).CheckProgress(ctx); err != nil {
+	if ready, abort, err := shutdown.CheckProgress(ctx); err != nil {
 		return false, abort, err
 	} else if !ready {
 		return false, false, nil
 	}
 
 	// Pod is terminated, we can now remove it
-	if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil {
+	if err := a.actionCtx.DeletePod(ctx, m.PodName, meta.DeleteOptions{}); err != nil {
 		if !k8sutil.IsNotFound(err) {
 			log.Error().Err(err).Msg("Unable to delete pod")
 			return false, false, nil