[Feature] Update Propagated flags

pkg/apis/deployment/v1/hashes.go

@@ -32,11 +32,15 @@ type DeploymentStatusHashes struct {
 
 type DeploymentStatusHashesEncryption struct {
 	Keys shared.HashList `json:"keys,omitempty"`
+
+	Propagated bool `json:"propagated,omitempty"`
 }
 
 type DeploymentStatusHashesTLS struct {
 	CA         *string         `json:"ca,omitempty"`
 	Truststore shared.HashList `json:"truststore,omitempty"`
+
+	Propagated bool `json:"propagated,omitempty"`
 }
 
 type DeploymentStatusHashesJWT struct {

pkg/apis/deployment/v1/plan.go

@@ -75,6 +75,8 @@ const (
 	ActionTypeRefreshTLSKeyfileCertificate ActionType = "RefreshTLSKeyfileCertificate"
 	// ActionTypeTLSKeyStatusUpdate update status with current data from deployment
 	ActionTypeTLSKeyStatusUpdate ActionType = "TLSKeyStatusUpdate"
+	// ActionTypeTLSPropagated change propagated flag
+	ActionTypeTLSPropagated ActionType = "TLSPropagated"
 	// ActionTypeUpdateTLSSNI update SNI inplace.
 	ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI"
 	// ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action.
@@ -101,6 +103,8 @@ const (
 	ActionTypeEncryptionKeyRefresh ActionType = "EncryptionKeyRefresh"
 	// ActionTypeEncryptionKeyStatusUpdate update status object with current encryption keys
 	ActionTypeEncryptionKeyStatusUpdate ActionType = "EncryptionKeyStatusUpdate"
+	// ActionTypeEncryptionKeyPropagated change propagated flag
+	ActionTypeEncryptionKeyPropagated ActionType = "EncryptionKeyPropagated"
 	// ActionTypeJWTStatusUpdate update status of JWT Secret
 	ActionTypeJWTStatusUpdate ActionType = "JWTStatusUpdate"
 	// ActionTypeJWTSetActive change active JWT key

pkg/deployment/pod/encryption.go

@@ -78,12 +78,12 @@ func GetEncryptionKey(secrets k8sutil.SecretInterface, name string) (string, []b
 
 func GetEncryptionKeyFromSecret(keyfile *core.Secret) (string, []byte, error) {
 	if len(keyfile.Data) == 0 {
-		return "", nil, nil
+		return "", nil, errors.Errorf("Current encryption key is not valid - missing data section")
 	}
 
 	d, ok := keyfile.Data[constants.SecretEncryptionKey]
 	if !ok {
-		return "", nil, nil
+		return "", nil, errors.Errorf("Current encryption key is not valid - missing field")
 	}
 
 	if len(d) != 32 {

pkg/deployment/reconcile/action_encryption_add.go

@@ -78,7 +78,7 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) {
 	}
 
 	secret := a.actionCtx.GetSpec().RocksDB.Encryption.GetKeySecretName()
-	if s, ok := a.action.Params["secret"]; ok {
+	if s, ok := a.action.Params[secretActionParam]; ok {
 		secret = s
 	}
 

pkg/deployment/reconcile/action_encryption_propagated.go

@@ -0,0 +1,71 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeEncryptionKeyPropagated, newEncryptionKeyPropagated)
+}
+
+func newEncryptionKeyPropagated(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &encryptionKeyPropagatedAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type encryptionKeyPropagatedAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *encryptionKeyPropagatedAction) Start(ctx context.Context) (bool, error) {
+	propagatedFlag, exists := a.action.Params[propagated]
+	if !exists {
+		a.log.Error().Msgf("Propagated flag is missing")
+		return true, nil
+	}
+
+	propagatedFlagBool := propagatedFlag == conditionTrue
+
+	if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		if s.Hashes.Encryption.Propagated != propagatedFlagBool {
+			s.Hashes.Encryption.Propagated = propagatedFlagBool
+			return true
+		}
+
+		return false
+	}); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_tls_propagated.go

@@ -0,0 +1,71 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeTLSPropagated, newTLSPropagated)
+}
+
+func newTLSPropagated(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &tlsPropagatedAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type tlsPropagatedAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *tlsPropagatedAction) Start(ctx context.Context) (bool, error) {
+	propagatedFlag, exists := a.action.Params[propagated]
+	if !exists {
+		a.log.Error().Msgf("Propagated flag is missing")
+		return true, nil
+	}
+
+	propagatedFlagBool := propagatedFlag == conditionTrue
+
+	if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		if s.Hashes.TLS.Propagated != propagatedFlagBool {
+			s.Hashes.TLS.Propagated = propagatedFlagBool
+			return true
+		}
+
+		return false
+	}); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/plan_builder.go

@@ -203,7 +203,7 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 
 	// Update status
 	if plan.IsEmpty() {
-		plan = pb.Apply(createEncryptionKeyStatusUpdate)
+		plan = pb.ApplySubPlan(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKeyStatusUpdate)
 	}
 
 	if plan.IsEmpty() {
@@ -226,19 +226,19 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 
 	// Add keys
 	if plan.IsEmpty() {
-		plan = pb.Apply(createEncryptionKey)
+		plan = pb.ApplySubPlan(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKey)
 	}
 
 	if plan.IsEmpty() {
 		plan = pb.Apply(createJWTKeyUpdate)
 	}
 
 	if plan.IsEmpty() {
-		plan = pb.Apply(createCARenewalPlan)
+		plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createCARenewalPlan)
 	}
 
 	if plan.IsEmpty() {
-		plan = pb.Apply(createCAAppendPlan)
+		plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createCAAppendPlan)
 	}
 
 	if plan.IsEmpty() {
@@ -251,25 +251,31 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 	}
 
 	if plan.IsEmpty() {
-		plan = pb.Apply(createRotateTLSServerSNIPlan)
+		plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createRotateTLSServerSNIPlan)
 	}
 
 	if plan.IsEmpty() {
 		plan = pb.Apply(createRestorePlan)
 	}
 
 	if plan.IsEmpty() {
-		plan = pb.Apply(createEncryptionKeyCleanPlan)
+		plan = pb.ApplySubPlan(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKeyCleanPlan)
 	}
 
 	if plan.IsEmpty() {
-		plan = pb.Apply(createCACleanPlan)
+		plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createCACleanPlan)
 	}
 
 	if plan.IsEmpty() {
 		plan = pb.Apply(createClusterOperationPlan)
 	}
 
+	// Final
+
+	if plan.IsEmpty() {
+		plan = pb.Apply(createTLSStatusPropagated)
+	}
+
 	// Return plan
 	return plan, true
 }
@@ -296,6 +302,11 @@ type planBuilder func(ctx context.Context,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan
 
+type planBuilderSubPlan func(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext, w WithPlanBuilder, plans ...planBuilder) api.Plan
+
 func NewWithPlanBuilder(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
@@ -313,6 +324,7 @@ func NewWithPlanBuilder(ctx context.Context,
 
 type WithPlanBuilder interface {
 	Apply(p planBuilder) api.Plan
+	ApplySubPlan(p planBuilderSubPlan, plans ...planBuilder) api.Plan
 }
 
 type withPlanBuilder struct {
@@ -325,6 +337,10 @@ type withPlanBuilder struct {
 	context      PlanBuilderContext
 }
 
+func (w withPlanBuilder) ApplySubPlan(p planBuilderSubPlan, plans ...planBuilder) api.Plan {
+	return p(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context, w, plans...)
+}
+
 func (w withPlanBuilder) Apply(p planBuilder) api.Plan {
 	return p(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context)
 }

pkg/deployment/reconcile/plan_builder_encryption.go

@@ -52,6 +52,43 @@ func skipEncryptionPlan(spec api.DeploymentSpec, status api.DeploymentStatus) bo
 	return false
 }
 
+func createEncryptionKeyStatusPropagatedFieldUpdate(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext, w WithPlanBuilder, builders ...planBuilder) api.Plan {
+	if skipEncryptionPlan(spec, status) {
+		return nil
+	}
+
+	var plan api.Plan
+
+	for _, builder := range builders {
+		if !plan.IsEmpty() {
+			continue
+		}
+
+		if p := w.Apply(builder); !p.IsEmpty() {
+			plan = append(plan, p...)
+		}
+	}
+
+	if plan.IsEmpty() {
+		return nil
+	}
+
+	if len(plan) == 1 && plan[0].Type == api.ActionTypeEncryptionKeyPropagated {
+		return plan
+	}
+
+	if status.Hashes.Encryption.Propagated {
+		plan = append(api.Plan{
+			api.NewAction(api.ActionTypeEncryptionKeyPropagated, api.ServerGroupUnknown, "", "Change propagated flag to false").AddParam(propagated, conditionFalse),
+		}, plan...)
+	}
+
+	return plan
+}
+
 func createEncryptionKey(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
@@ -85,17 +122,24 @@ func createEncryptionKey(ctx context.Context,
 		keyfolder.Data = map[string][]byte{}
 	}
 
-	_, ok := keyfolder.Data[name]
-	if !ok {
-		return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "")}
+	if status.Hashes.Encryption.Propagated {
+		_, ok := keyfolder.Data[name]
+		if !ok {
+			return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "")}
+		}
 	}
 
-	plan, _ := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder)
-
+	plan, failed := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder)
 	if !plan.IsEmpty() {
 		return plan
 	}
 
+	if !failed && !status.Hashes.Encryption.Propagated {
+		return api.Plan{
+			api.NewAction(api.ActionTypeEncryptionKeyPropagated, api.ServerGroupUnknown, "", "Change propagated flag to true").AddParam(propagated, conditionTrue),
+		}
+	}
+
 	return api.Plan{}
 }
 
@@ -152,17 +196,11 @@ func createEncryptionKeyCleanPlan(ctx context.Context,
 		return nil
 	}
 
-	plan, failed := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder)
-
-	if failed {
-		log.Info().Msgf("Unable to continue with encryption until all servers are ready")
+	if !status.Hashes.Encryption.Propagated {
 		return nil
 	}
 
-	if len(plan) != 0 {
-		log.Info().Msgf("Unable to continue with encryption until all servers report state or gonna be upToDate")
-		return nil
-	}
+	var plan api.Plan
 
 	if len(keyfolder.Data) <= 1 {
 		return nil