[Feature] Optional Graceful Restart

CHANGELOG.md

@@ -22,6 +22,7 @@
 - (Improvement) Cleanout calculation - picks members with the lowest number of shards
 - (Improvement) Add new field to CR for more precise calculation of DC2DC replication progress
 - (Maintenance) Bump GO Modules
+- (Feature) Optional Graceful Restart
 
 ## [1.2.24](https://github.com/arangodb/kube-arangodb/tree/1.2.24) (2023-01-25)
 - (Bugfix) Fix deployment creation on ARM64

README.md

@@ -64,6 +64,8 @@ Feature-wise production readiness table:
 | Version Check                           | 1.1.4            | >= 3.6.0         | Community, Enterprise | 1.1.4      | Alpha        | False   | --deployment.feature.upgrade-version-check            | N/A                                                                      |
 | Version Check                           | 1.2.23           | >= 3.6.0         | Community, Enterprise | 1.1.4      | Production   | True    | --deployment.feature.upgrade-version-check            | N/A                                                                      |
 | Operator Maintenance Management Support | 1.2.0            | >= 3.6.0         | Community, Enterprise | 1.0.7      | Production   | True    | --deployment.feature.maintenance                      | N/A                                                                      |
+| Graceful Restart                        | 1.2.5            | >= 3.6.0         | Community, Enterprise | 1.0.7      | Production   | True    | --deployment.feature.graceful-shutdown                | N/A                                                                      |
+| Optional Graceful Restart               | 1.2.25           | >= 3.6.0         | Community, Enterprise | 1.2.5      | Beta         | True    | --deployment.feature.optional-graceful-shutdown       | N/A                                                                      |
 | Operator Internal Metrics Exporter      | 1.2.0            | >= 3.6.0         | Community, Enterprise | 1.2.0      | Production   | True    | --deployment.feature.metrics-exporter                 | N/A                                                                      |
 | Operator Ephemeral Volumes              | 1.2.2            | >= 3.7.0         | Community, Enterprise | 1.2.2      | Alpha        | False   | --deployment.feature.ephemeral-volumes                | N/A                                                                      |
 | Spec Default Restore                    | 1.2.21           | >= 3.7.0         | Community, Enterprise | 1.2.21     | Beta         | True    | --deployment.feature.deployment-spec-defaults-restore | If set to False Operator will not change ArangoDeployment Spec           |

go.mod

@@ -20,6 +20,8 @@ replace (
 	k8s.io/component-base => k8s.io/component-base v0.22.15
 	k8s.io/kubernetes => k8s.io/kubernetes v0.22.15
 	k8s.io/metrics => k8s.io/metrics v0.22.15
+
+	gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

go.sum

@@ -972,9 +972,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/deployment/features/graceful.go

@@ -22,6 +22,7 @@ package features
 
 func init() {
 	registerFeature(gracefulShutdown)
+	registerFeature(optionalGracefulShutdown)
 }
 
 var gracefulShutdown = &feature{
@@ -33,6 +34,19 @@ var gracefulShutdown = &feature{
 	hidden:             true,
 }
 
+var optionalGracefulShutdown = &feature{
+	name:               "optional-graceful-shutdown",
+	description:        "Define graceful shutdown, using finalizers, is optional and can fail in case of connection issues",
+	version:            "3.6.0",
+	enterpriseRequired: false,
+	enabledByDefault:   false,
+	hidden:             true,
+}
+
 func GracefulShutdown() Feature {
 	return gracefulShutdown
 }
+
+func OptionalGracefulShutdown() Feature {
+	return optionalGracefulShutdown
+}

pkg/deployment/reconcile/helper_shutdown.go

@@ -79,7 +79,7 @@ func getShutdownHelper(a actionImpl) (ActionCore, api.MemberStatus, bool) {
 	}
 
 	if features.GracefulShutdown().Enabled() {
-		return shutdownHelperAPI{actionImpl: a, memberStatus: m}, m, true
+		return getShutdownHelperAPI(a, m), m, true
 	}
 
 	serverGroup := a.actionCtx.GetSpec().GetServerGroupSpec(a.action.Group)
@@ -88,10 +88,40 @@ func getShutdownHelper(a actionImpl) (ActionCore, api.MemberStatus, bool) {
 	case api.ServerGroupShutdownMethodDelete:
 		return shutdownHelperDelete{actionImpl: a, memberStatus: m}, m, true
 	default:
-		return shutdownHelperAPI{actionImpl: a, memberStatus: m}, m, true
+		return getShutdownHelperAPI(a, m), m, true
 	}
 }
 
+func getShutdownHelperAPI(a actionImpl, member api.MemberStatus) ActionCore {
+	act := shutdownHelperAPI{actionImpl: a, memberStatus: member}
+
+	if !features.OptionalGracefulShutdown().Enabled() {
+		return act
+	}
+
+	return shutdownHelperOptionalAPI{action: act}
+}
+
+type shutdownHelperOptionalAPI struct {
+	action shutdownHelperAPI
+}
+
+func (s shutdownHelperOptionalAPI) Start(ctx context.Context) (bool, error) {
+	return false, nil
+}
+
+func (s shutdownHelperOptionalAPI) CheckProgress(ctx context.Context) (bool, bool, error) {
+	if done, abort, err := s.action.CheckProgress(ctx); err != nil || abort || done {
+		return done, abort, err
+	}
+
+	if _, err := s.action.Start(ctx); err != nil {
+		return false, false, nil
+	}
+
+	return false, false, nil
+}
+
 type shutdownHelperAPI struct {
 	actionImpl
 	memberStatus api.MemberStatus