GT-6 Add ArangoSync TLS based rotation

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- (Feature) Add ArangoSync TLS based rotation
 
 ## [1.2.13](https://github.com/arangodb/kube-arangodb/tree/1.2.13) (2022-06-07)
 - (Bugfix) Fix arangosync members state inspection

pkg/apis/shared/constants.go

@@ -33,6 +33,8 @@ const (
 	ArangoExporterInternalEndpointV2    = "/_admin/metrics/v2"
 	ArangoExporterDefaultEndpoint       = "/metrics"
 
+	ArangoSyncStatusEndpoint = "/_api/version"
+
 	// K8s constants
 	ClusterIPNone       = "None"
 	TopologyKeyHostname = "kubernetes.io/hostname"

pkg/deployment/reconcile/plan_builder_tls.go

@@ -30,21 +30,20 @@ import (
 	"reflect"
 	"time"
 
-	memberTls "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/tls"
-
-	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
-
-	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
-	"github.com/arangodb/kube-arangodb/pkg/util/constants"
-	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
-
 	"github.com/arangodb/go-driver"
+
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
+	memberTls "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/tls"
+
 	"github.com/rs/zerolog"
 )
 
@@ -286,6 +285,42 @@ func createCACleanPlan(ctx context.Context,
 	return nil
 }
 
+func createKeyfileRenewalPlanSynced(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	planCtx PlanBuilderContext) api.Plan {
+
+	if !spec.Sync.IsEnabled() || !spec.Sync.TLS.IsSecure() {
+		return nil
+	}
+
+	var plan api.Plan
+	group := api.ServerGroupSyncMasters
+
+	for _, statusMember := range status.Members.AsListInGroup(group) {
+		member := statusMember.Member
+
+		if !plan.IsEmpty() {
+			return nil
+		}
+
+		cache, ok := planCtx.ACS().ClusterCache(member.ClusterID)
+		if !ok {
+			continue
+		}
+
+		lCtx, c := context.WithTimeout(ctx, 500*time.Millisecond)
+		defer c()
+
+		if renew, _ := keyfileRenewalRequired(lCtx, log, apiObject, spec.Sync.TLS, spec, cache, planCtx, group, member, api.TLSRotateModeRecreate); renew {
+			log.Info().Msg("Renewal of keyfile required - Recreate (sync master)")
+			plan = append(plan, tlsRotateConditionAction(group, member.ID, "Restart sync master after keyfile removal"))
+		}
+	}
+
+	return plan
+}
+
 func createKeyfileRenewalPlanDefault(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
@@ -314,8 +349,8 @@ func createKeyfileRenewalPlanDefault(ctx context.Context,
 			lCtx, c := context.WithTimeout(ctx, 500*time.Millisecond)
 			defer c()
 
-			if renew, _ := keyfileRenewalRequired(lCtx, log, apiObject, spec, cache, planCtx, group, member, api.TLSRotateModeRecreate); renew {
-				log.Info().Msg("Renewal of keyfile required - Recreate")
+			if renew, _ := keyfileRenewalRequired(lCtx, log, apiObject, spec.TLS, spec, cache, planCtx, group, member, api.TLSRotateModeRecreate); renew {
+				log.Info().Msg("Renewal of keyfile required - Recreate (server)")
 				plan = append(plan, tlsRotateConditionAction(group, member.ID, "Restart server after keyfile removal"))
 			}
 		}
@@ -350,8 +385,8 @@ func createKeyfileRenewalPlanInPlace(ctx context.Context,
 			lCtx, c := context.WithTimeout(ctx, 500*time.Millisecond)
 			defer c()
 
-			if renew, recreate := keyfileRenewalRequired(lCtx, log, apiObject, spec, cache, planCtx, group, member, api.TLSRotateModeInPlace); renew {
-				log.Info().Msg("Renewal of keyfile required - InPlace")
+			if renew, recreate := keyfileRenewalRequired(lCtx, log, apiObject, spec.TLS, spec, cache, planCtx, group, member, api.TLSRotateModeInPlace); renew {
+				log.Info().Msg("Renewal of keyfile required - InPlace (server)")
 				if recreate {
 					plan = append(plan, actions.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member, "Remove server keyfile and enforce renewal"))
 				}
@@ -376,12 +411,16 @@ func createKeyfileRenewalPlan(ctx context.Context,
 	gCtx, c := context.WithTimeout(ctx, 2*time.Second)
 	defer c()
 
+	plan := createKeyfileRenewalPlanSynced(gCtx, log, apiObject, spec, status, planCtx)
+
 	switch createKeyfileRenewalPlanMode(spec, status) {
 	case api.TLSRotateModeInPlace:
-		return createKeyfileRenewalPlanInPlace(gCtx, log, apiObject, spec, status, planCtx)
+		plan = append(plan, createKeyfileRenewalPlanInPlace(gCtx, log, apiObject, spec, status, planCtx)...)
 	default:
-		return createKeyfileRenewalPlanDefault(gCtx, log, apiObject, spec, status, planCtx)
+		plan = append(plan, createKeyfileRenewalPlanDefault(gCtx, log, apiObject, spec, status, planCtx)...)
 	}
+
+	return plan
 }
 
 func createKeyfileRenewalPlanMode(
@@ -419,6 +458,9 @@ func createKeyfileRenewalPlanMode(
 
 func checkServerValidCertRequest(ctx context.Context, context PlanBuilderContext, apiObject k8sutil.APIObject, group api.ServerGroup, member api.MemberStatus, ca resources.Certificates) (*tls.ConnectionState, error) {
 	endpoint := fmt.Sprintf("https://%s:%d", k8sutil.CreatePodDNSNameWithDomain(apiObject, context.GetSpec().ClusterDomain, group.AsRole(), member.ID), shared.ArangoPort)
+	if group == api.ServerGroupSyncMasters {
+		endpoint = fmt.Sprintf("https://%s:%d%s", k8sutil.CreatePodDNSNameWithDomain(apiObject, context.GetSpec().ClusterDomain, group.AsRole(), member.ID), shared.ArangoSyncMasterPort, shared.ArangoSyncStatusEndpoint)
+	}
 
 	tlsConfig := &tls.Config{
 		RootCAs: ca.AsCertPool(),
@@ -452,12 +494,13 @@ func checkServerValidCertRequest(ctx context.Context, context PlanBuilderContext
 	return resp.TLS, nil
 }
 
+// keyfileRenewalRequired checks if a keyfile renewal is required and if recreation should be made
 func keyfileRenewalRequired(ctx context.Context,
-	log zerolog.Logger, apiObject k8sutil.APIObject,
+	log zerolog.Logger, apiObject k8sutil.APIObject, tls api.TLSSpec,
 	spec api.DeploymentSpec, cachedStatus inspectorInterface.Inspector,
 	context PlanBuilderContext,
 	group api.ServerGroup, member api.MemberStatus, mode api.TLSRotateMode) (bool, bool) {
-	if !spec.TLS.IsSecure() {
+	if !tls.IsSecure() {
 		return false, false
 	}
 
@@ -469,15 +512,15 @@ func keyfileRenewalRequired(ctx context.Context,
 		return false, false
 	}
 
-	caSecret, exists := cachedStatus.Secret().V1().GetSimple(spec.TLS.GetCASecretName())
+	caSecret, exists := cachedStatus.Secret().V1().GetSimple(tls.GetCASecretName())
 	if !exists {
-		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists")
+		log.Warn().Str("secret", tls.GetCASecretName()).Msg("CA Secret does not exists")
 		return false, false
 	}
 
 	ca, _, err := resources.GetKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName)
 	if err != nil {
-		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert")
+		log.Warn().Err(err).Str("secret", tls.GetCASecretName()).Msg("CA Secret does not contains Cert")
 		return false, false
 	}
 
@@ -487,13 +530,13 @@ func keyfileRenewalRequired(ctx context.Context,
 		case *url.Error:
 			switch v.Err.(type) {
 			case x509.UnknownAuthorityError, x509.CertificateInvalidError:
-				log.Debug().Err(v.Err).Str("type", reflect.TypeOf(v.Err).String()).Msg("Validation of server cert failed")
+				log.Debug().Err(v.Err).Str("type", reflect.TypeOf(v.Err).String()).Msgf("Validation of cert for %s failed, renewal is required", memberName)
 				return true, true
 			default:
-				log.Debug().Err(v.Err).Str("type", reflect.TypeOf(v.Err).String()).Msg("Validation of server cert failed")
+				log.Debug().Err(v.Err).Str("type", reflect.TypeOf(v.Err).String()).Msgf("Validation of cert for %s failed, but cert looks fine - continuing", memberName)
 			}
 		default:
-			log.Debug().Err(err).Str("type", reflect.TypeOf(err).String()).Msg("Validation of server cert failed")
+			log.Debug().Err(err).Str("type", reflect.TypeOf(err).String()).Msgf("Validation of cert for %s failed, will try again next time", memberName)
 		}
 		return false, false
 	}
@@ -514,7 +557,13 @@ func keyfileRenewalRequired(ctx context.Context,
 		}
 
 		// Verify AltNames
-		altNames, err := memberTls.GetServerAltNames(apiObject, spec, spec.TLS, service, group, member)
+		var altNames memberTls.KeyfileInput
+		if group.IsArangosync() {
+			altNames, err = memberTls.GetSyncAltNames(apiObject, spec, tls, group, member)
+		} else {
+			altNames, err = memberTls.GetServerAltNames(apiObject, spec, tls, service, group, member)
+		}
+
 		if err != nil {
 			log.Warn().Msg("Unable to render alt names")
 			return false, false
@@ -535,7 +584,7 @@ func keyfileRenewalRequired(ctx context.Context,
 	}
 
 	// Ensure secret is propagated only on 3.7.0+ enterprise and inplace mode
-	if mode == api.TLSRotateModeInPlace {
+	if mode == api.TLSRotateModeInPlace && group.IsArangod() {
 		conn, err := context.GetServerClient(ctx, group, member.ID)
 		if err != nil {
 			log.Warn().Err(err).Msg("Unable to get client")

pkg/deployment/resources/pod_creator.go

@@ -26,7 +26,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
-	"net/url"
 	"path/filepath"
 	"strconv"
 	"sync"
@@ -530,22 +529,11 @@ func (r *Resources) createPodForMember(ctx context.Context, cachedStatus inspect
 			// Create TLS secret
 			tlsKeyfileSecretName := k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID)
 
-			names, err := tls.GetAltNames(spec.Sync.TLS)
+			names, err := tls.GetSyncAltNames(apiObject, spec, spec.Sync.TLS, group, m)
 			if err != nil {
 				return errors.WithStack(errors.Wrapf(err, "Failed to render alt names"))
 			}
 
-			names.AltNames = append(names.AltNames,
-				k8sutil.CreateSyncMasterClientServiceName(apiObject.GetName()),
-				k8sutil.CreateSyncMasterClientServiceDNSNameWithDomain(apiObject, spec.ClusterDomain),
-				k8sutil.CreatePodDNSNameWithDomain(apiObject, spec.ClusterDomain, role, m.ID),
-			)
-			masterEndpoint := spec.Sync.ExternalAccess.ResolveMasterEndpoint(k8sutil.CreateSyncMasterClientServiceDNSNameWithDomain(apiObject, spec.ClusterDomain), shared.ArangoSyncMasterPort)
-			for _, ep := range masterEndpoint {
-				if u, err := url.Parse(ep); err == nil {
-					names.AltNames = append(names.AltNames, u.Hostname())
-				}
-			}
 			owner := apiObject.AsOwner()
 			_, err = createTLSServerCertificate(ctx, log, cachedStatus, cachedStatus.SecretsModInterface().V1(), names, spec.Sync.TLS, tlsKeyfileSecretName, &owner)
 			if err != nil && !k8sutil.IsAlreadyExists(err) {

pkg/util/k8sutil/tls/tls.go

@@ -21,9 +21,13 @@
 package tls
 
 import (
+	"net/url"
+
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+
 	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -86,3 +90,25 @@ func GetServerAltNames(deployment meta.Object, spec api.DeploymentSpec, tls api.
 
 	return k, nil
 }
+
+func GetSyncAltNames(deployment meta.Object, spec api.DeploymentSpec, tls api.TLSSpec, group api.ServerGroup, member api.MemberStatus) (KeyfileInput, error) {
+	k, err := GetAltNames(tls)
+	if err != nil {
+		return k, errors.WithStack(err)
+	}
+
+	k.AltNames = append(k.AltNames,
+		k8sutil.CreateSyncMasterClientServiceName(deployment.GetName()),
+		k8sutil.CreateSyncMasterClientServiceDNSNameWithDomain(deployment, spec.ClusterDomain),
+		k8sutil.CreatePodDNSNameWithDomain(deployment, spec.ClusterDomain, group.AsRole(), member.ID),
+	)
+
+	masterEndpoint := spec.Sync.ExternalAccess.ResolveMasterEndpoint(k8sutil.CreateSyncMasterClientServiceDNSNameWithDomain(deployment, spec.ClusterDomain), shared.ArangoSyncMasterPort)
+	for _, ep := range masterEndpoint {
+		if u, err := url.Parse(ep); err == nil {
+			k.AltNames = append(k.AltNames, u.Hostname())
+		}
+	}
+
+	return k, nil
+}