[Feature] Graceful Shutdown

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Split & Unify Lifecycle management functionality
 - Drop support for ArangoDB <= 3.5 (versions already EOL)
 - Add new admin commands to fetch agency dump and agency state
+- Add Graceful shutdown as finalizer (supports kubectl delete)
 
 ## [1.2.4](https://github.com/arangodb/kube-arangodb/tree/1.2.4) (2021-10-22)
 - Replace `beta.kubernetes.io/arch` Pod label with `kubernetes.io/arch` using Silent Rotation

go.mod

@@ -3,7 +3,7 @@ module github.com/arangodb/kube-arangodb
 go 1.16
 
 replace (
-	github.com/arangodb/go-driver => github.com/arangodb/go-driver v0.0.0-20210621075908-e7a6fa0cbd18
+	github.com/arangodb/go-driver => github.com/arangodb/go-driver v1.2.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring => github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.46.0
 	github.com/prometheus-operator/prometheus-operator/pkg/client => github.com/prometheus-operator/prometheus-operator/pkg/client v0.46.0
 	github.com/stretchr/testify => github.com/stretchr/testify v1.5.1
@@ -25,8 +25,8 @@ replace (
 require (
 	github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680
 	github.com/arangodb/arangosync-client v0.7.0
-	github.com/arangodb/go-driver v0.0.0-20210621075908-e7a6fa0cbd18
-	github.com/arangodb/go-driver/v2 v2.0.0-20211001173946-eafa9b638e13
+	github.com/arangodb/go-driver v1.2.1
+	github.com/arangodb/go-driver/v2 v2.0.0-20211021031401-d92dcd5a4c83
 	github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9

go.sum

@@ -44,8 +44,10 @@ github.com/arangodb/arangosync-client v0.7.0 h1:3vLOVnMyr5vGlPA0OHxJL9Wyy49JJwN0
 github.com/arangodb/arangosync-client v0.7.0/go.mod h1:g+JcxH3C63wKaJPnPr9nggYoGbt/bYCWpfcRG0NSodY=
 github.com/arangodb/go-driver v0.0.0-20210621075908-e7a6fa0cbd18 h1:3J0tqp5eQ8ptGOeeu7vo92RKf24bOA7MFy0z3uPiTWg=
 github.com/arangodb/go-driver v0.0.0-20210621075908-e7a6fa0cbd18/go.mod h1:3NUekcRLpgheFIGEwcOvxilEW73MV1queNKW58k7sdc=
-github.com/arangodb/go-driver/v2 v2.0.0-20211001173946-eafa9b638e13 h1:5egTRo3Met3xXUVj/Pbn1gXeY2C4bQZycJoHSnndfig=
-github.com/arangodb/go-driver/v2 v2.0.0-20211001173946-eafa9b638e13/go.mod h1:X3uG4XbfQS35AjsFJLwNLyA6UZofNV5ufe2KoNxcMO0=
+github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8=
+github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY=
+github.com/arangodb/go-driver/v2 v2.0.0-20211021031401-d92dcd5a4c83 h1:PCbi3alUFastUw6InBKGEXqniveJJcQuMYspubJMRS8=
+github.com/arangodb/go-driver/v2 v2.0.0-20211021031401-d92dcd5a4c83/go.mod h1:B8byYwvt1mDOQzpjiMuDTP5jOif/Y5dcEJtkdvPB7HY=
 github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21 h1:+W7D5ttxi/Ygh/39vialtypE23p9KI7P0J2qtoqUV4w=
 github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21/go.mod h1:RkPIG6JJ2pcJUoymc18NxAJGraZd+iAEVnOTDjZey/w=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
@@ -184,6 +186,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

lifecycle.go

@@ -25,10 +25,16 @@ package main
 import (
 	"context"
 	"io"
+	"net"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/backup/utils"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/retry"
+
 	"github.com/arangodb/kube-arangodb/pkg/version"
 
 	"github.com/spf13/cobra"
@@ -54,6 +60,10 @@ var (
 		Run:    cmdLifecyclePreStopRunFinalizer,
 		Hidden: true,
 	}
+	cmdLifecyclePreStopPort = &cobra.Command{
+		Use:    "port",
+		Hidden: true,
+	}
 	cmdLifecycleCopy = &cobra.Command{
 		Use:    "copy",
 		Run:    cmdLifecycleCopyRun,
@@ -68,7 +78,15 @@ var (
 func init() {
 	cmdMain.AddCommand(cmdLifecycle)
 
-	cmdLifecyclePreStop.AddCommand(cmdLifecyclePreStopFinalizers)
+	var preStopPort cmdLifecyclePreStopRunPort
+
+	cmdLifecyclePreStopPort.RunE = preStopPort.run
+
+	f := cmdLifecyclePreStopPort.Flags()
+
+	f.DurationVar(&preStopPort.timeout, "timeout", 6*60*time.Minute, "PreStopTimeout")
+
+	cmdLifecyclePreStop.AddCommand(cmdLifecyclePreStopFinalizers, cmdLifecyclePreStopPort)
 
 	cmdLifecycle.AddCommand(cmdLifecyclePreStop)
 	cmdLifecycle.AddCommand(cmdLifecycleCopy)
@@ -162,3 +180,63 @@ func cmdLifecycleCopyRun(cmd *cobra.Command, args []string) {
 
 	cliLog.Info().Msgf("Executable copied to %s", targetPath)
 }
+
+type cmdLifecyclePreStopRunPort struct {
+	timeout time.Duration
+}
+
+// Wait until port 8529 is closed.
+func (c *cmdLifecyclePreStopRunPort) run(cmd *cobra.Command, args []string) error {
+	address := net.JoinHostPort("127.0.0.1", strconv.Itoa(k8sutil.ArangoPort))
+
+	// Get environment
+	namespace := os.Getenv(constants.EnvOperatorPodNamespace)
+	if len(namespace) == 0 {
+		cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodNamespace)
+	}
+	name := os.Getenv(constants.EnvOperatorPodName)
+	if len(name) == 0 {
+		cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName)
+	}
+
+	// Create kubernetes client
+	kubecli, err := k8sutil.NewKubeClient()
+	if err != nil {
+		cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client")
+	}
+
+	pods := kubecli.CoreV1().Pods(namespace)
+
+	recentErrors := 0
+
+	return retry.NewTimeout(func() error {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+
+		if err != nil {
+			return retry.Interrput()
+		}
+
+		conn.Close()
+
+		p, err := pods.Get(context.Background(), name, metav1.GetOptions{})
+		if k8sutil.IsNotFound(err) {
+			cliLog.Warn().Msg("Pod not found")
+			return nil
+		} else if err != nil {
+			recentErrors++
+			cliLog.Error().Err(err).Msg("Failed to get pod")
+			if recentErrors > 20 {
+				cliLog.Fatal().Err(err).Msg("Too many recent errors")
+				return nil
+			}
+		} else {
+			// We got our pod
+			finalizers := utils.StringList(p.GetFinalizers())
+			if !finalizers.Has(constants.FinalizerPodGracefulShutdown) {
+				return retry.Interrput()
+			}
+		}
+
+		return nil
+	}).Timeout(125*time.Millisecond, c.timeout)
+}

pkg/apis/deployment/v1/plan.go

@@ -73,6 +73,8 @@ const (
 	ActionTypeShutdownMember ActionType = "ShutdownMember"
 	// ActionTypeResignLeadership causes a member to resign leadership.
 	ActionTypeResignLeadership ActionType = "ResignLeadership"
+	// ActionTypeKillMemberPod causes a pod to get delete request. It also waits until Delay finalizer will be removed.
+	ActionTypeKillMemberPod ActionType = "KillMemberPod"
 	// ActionTypeRotateMember causes a member to be shutdown and have it's pod removed.
 	ActionTypeRotateMember ActionType = "RotateMember"
 	// ActionTypeRotateStartMember causes a member to be shutdown and have it's pod removed. Do not wait to pod recover.

pkg/apis/deployment/v1/server_group.go

@@ -148,6 +148,8 @@ func (g ServerGroup) DefaultTerminationGracePeriod() time.Duration {
 		return time.Minute
 	case ServerGroupDBServers:
 		return time.Hour
+	case ServerGroupCoordinators:
+		return time.Hour
 	default:
 		return time.Second * 30
 	}

pkg/apis/deployment/v1/server_group_spec.go

@@ -682,3 +682,16 @@ func (s *ServerGroupSpec) GetEntrypoint(defaultEntrypoint string) string {
 
 	return *s.Entrypoint
 }
+
+// GetShutdownDelay returns defined or default Group ShutdownDelay in seconds
+func (s ServerGroupSpec) GetShutdownDelay(group ServerGroup) int {
+	if s.ShutdownDelay == nil {
+		switch group {
+		case ServerGroupCoordinators:
+			return 3
+		default:
+			return 0
+		}
+	}
+	return *s.ShutdownDelay
+}

pkg/apis/deployment/v2alpha1/deployment_metrics_spec.go

@@ -41,6 +41,7 @@ func (m MetricsMode) New() *MetricsMode {
 	return &m
 }
 
+// deprecated
 func (m MetricsMode) GetMetricsEndpoint() string {
 	switch m {
 	case MetricsModeInternal:
@@ -51,9 +52,12 @@ func (m MetricsMode) GetMetricsEndpoint() string {
 }
 
 const (
+	// deprecated
 	// MetricsModeExporter exporter mode for old exporter type
 	MetricsModeExporter MetricsMode = "exporter"
-	MetricsModeSidecar  MetricsMode = "sidecar"
+	// deprecated
+	MetricsModeSidecar MetricsMode = "sidecar"
+	// deprecated
 	MetricsModeInternal MetricsMode = "internal"
 )
 
@@ -67,12 +71,14 @@ func (m *MetricsMode) Get() MetricsMode {
 
 // MetricsSpec contains spec for arangodb exporter
 type MetricsSpec struct {
-	Enabled        *bool                     `json:"enabled,omitempty"`
+	Enabled *bool `json:"enabled,omitempty"`
+	// deprecated
 	Image          *string                   `json:"image,omitempty"`
 	Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"`
 	Resources      v1.ResourceRequirements   `json:"resources,omitempty"`
-	Mode           *MetricsMode              `json:"mode,omitempty"`
-	TLS            *bool                     `json:"tls,omitempty"`
+	// deprecated
+	Mode *MetricsMode `json:"mode,omitempty"`
+	TLS  *bool        `json:"tls,omitempty"`
 
 	ServiceMonitor *MetricsServiceMonitorSpec `json:"serviceMonitor,omitempty"`
 
@@ -100,11 +106,13 @@ func (s *MetricsSpec) IsEnabled() bool {
 	return util.BoolOrDefault(s.Enabled, false)
 }
 
+// deprecated
 // HasImage returns whether a image was specified or not
 func (s *MetricsSpec) HasImage() bool {
 	return s.Image != nil
 }
 
+// deprecated
 // GetImage returns the Image or empty string
 func (s *MetricsSpec) GetImage() string {
 	return util.StringOrDefault(s.Image)

pkg/apis/deployment/v2alpha1/plan.go

@@ -73,6 +73,8 @@ const (
 	ActionTypeShutdownMember ActionType = "ShutdownMember"
 	// ActionTypeResignLeadership causes a member to resign leadership.
 	ActionTypeResignLeadership ActionType = "ResignLeadership"
+	// ActionTypeKillMemberPod causes a pod to get delete request. It also waits until Delay finalizer will be removed.
+	ActionTypeKillMemberPod ActionType = "KillMemberPod"
 	// ActionTypeRotateMember causes a member to be shutdown and have it's pod removed.
 	ActionTypeRotateMember ActionType = "RotateMember"
 	// ActionTypeRotateStartMember causes a member to be shutdown and have it's pod removed. Do not wait to pod recover.

pkg/apis/deployment/v2alpha1/server_group.go

@@ -148,6 +148,8 @@ func (g ServerGroup) DefaultTerminationGracePeriod() time.Duration {
 		return time.Minute
 	case ServerGroupDBServers:
 		return time.Hour
+	case ServerGroupCoordinators:
+		return time.Hour
 	default:
 		return time.Second * 30
 	}

pkg/apis/deployment/v2alpha1/server_group_containers.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+const (
+	ServerGroupReservedContainerNameServer   = "server"
+	ServerGroupReservedContainerNameExporter = "exporter"
+)
+
+func IsReservedServerGroupContainerName(name string) bool {
+	switch name {
+	case ServerGroupReservedContainerNameServer, ServerGroupReservedContainerNameExporter:
+		return true
+	default:
+		return false
+	}
+}

pkg/apis/deployment/v2alpha1/server_group_spec.go

@@ -682,3 +682,16 @@ func (s *ServerGroupSpec) GetEntrypoint(defaultEntrypoint string) string {
 
 	return *s.Entrypoint
 }
+
+// GetShutdownDelay returns defined or default Group ShutdownDelay in seconds
+func (s ServerGroupSpec) GetShutdownDelay(group ServerGroup) int {
+	if s.ShutdownDelay == nil {
+		switch group {
+		case ServerGroupCoordinators:
+			return 3
+		default:
+			return 0
+		}
+	}
+	return *s.ShutdownDelay
+}