feat(kuma-cp): separate CA for Envoy Admin communication

pkg/core/bootstrap/bootstrap.go

@@ -115,19 +115,14 @@ func buildRuntime(appCtx context.Context, cfg kuma_cp.Config) (core_runtime.Runt
 	if cfg.Mode == config_core.Global {
 		builder.WithEnvoyAdminClient(admin.NewKDSEnvoyAdminClient(
 			builder.KDSContext().EnvoyAdminRPCs,
-			cfg.Store.Type == store.KubernetesStore))
+			cfg.Store.Type == store.KubernetesStore,
+		))
 	} else {
-		envoyAdminClient, err := admin.NewEnvoyAdminClient(
+		builder.WithEnvoyAdminClient(admin.NewEnvoyAdminClient(
 			builder.ResourceManager(),
 			builder.CaManagers(),
-			builder.Config().DpServer.TlsCertFile,
-			builder.Config().DpServer.TlsKeyFile,
 			builder.Config().GetEnvoyAdminPort(),
-		)
-		if err != nil {
-			return nil, err
-		}
-		builder.WithEnvoyAdminClient(envoyAdminClient)
+		))
 	}
 
 	builder.WithAccess(core_runtime.Access{

pkg/core/xds/types.go

@@ -11,6 +11,7 @@ import (
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
+	util_tls "github.com/kumahq/kuma/pkg/tls"
 	envoy_common "github.com/kumahq/kuma/pkg/xds/envoy"
 	"github.com/kumahq/kuma/pkg/xds/envoy/names"
 	xds_tls "github.com/kumahq/kuma/pkg/xds/envoy/tls"
@@ -135,13 +136,14 @@ const (
 // Proxy contains required data for generating XDS config that is specific to a data plane proxy.
 // The data that is specific for the whole mesh should go into MeshContext.
 type Proxy struct {
-	Id          ProxyId
-	APIVersion  envoy_common.APIVersion // todo(jakubdyszkiewicz) consider moving APIVersion here. pkg/core should not depend on pkg/xds. It should be other way around.
-	Dataplane   *core_mesh.DataplaneResource
-	ZoneIngress *core_mesh.ZoneIngressResource
-	Metadata    *DataplaneMetadata
-	Routing     Routing
-	Policies    MatchedPolicies
+	Id                  ProxyId
+	APIVersion          envoy_common.APIVersion // todo(jakubdyszkiewicz) consider moving APIVersion here. pkg/core should not depend on pkg/xds. It should be other way around.
+	Dataplane           *core_mesh.DataplaneResource
+	ZoneIngress         *core_mesh.ZoneIngressResource
+	Metadata            *DataplaneMetadata
+	Routing             Routing
+	Policies            MatchedPolicies
+	EnvoyAdminMTLSCerts ServerSideMTLSCerts
 
 	// SecretsTracker allows us to track when a generator references a secret so
 	// we can be sure to include only those secrets later on.
@@ -153,6 +155,11 @@ type Proxy struct {
 	ZoneIngressProxy *ZoneIngressProxy
 }
 
+type ServerSideMTLSCerts struct {
+	CaPEM      []byte
+	ServerPair util_tls.KeyPair
+}
+
 type identityCertRequest struct {
 	meshName string
 }

pkg/defaults/components.go

@@ -24,27 +24,34 @@ import (
 var log = core.Log.WithName("defaults")
 
 func Setup(runtime runtime.Runtime) error {
-	if runtime.Config().Mode == config_core.Zone { // Don't run defaults in Zone (it's done in Global)
-		return nil
-	}
-	defaultsComponent := NewDefaultsComponent(runtime.Config().Defaults, runtime.Config().Mode, runtime.Config().Environment, runtime.ResourceManager(), runtime.ResourceStore())
+	if runtime.Config().Mode != config_core.Zone { // Don't run defaults in Zone (it's done in Global)
+		defaultsComponent := NewDefaultsComponent(runtime.Config().Defaults, runtime.Config().Mode, runtime.Config().Environment, runtime.ResourceManager(), runtime.ResourceStore())
+
+		zoneIngressSigningKeyManager := tokens.NewSigningKeyManager(runtime.ResourceManager(), zoneingress.ZoneIngressSigningKeyPrefix)
+		if err := runtime.Add(tokens.NewDefaultSigningKeyComponent(
+			zoneIngressSigningKeyManager,
+			log.WithValues("secretPrefix", zoneingress.ZoneIngressSigningKeyPrefix))); err != nil {
+			return err
+		}
 
-	zoneIngressSigningKeyManager := tokens.NewSigningKeyManager(runtime.ResourceManager(), zoneingress.ZoneIngressSigningKeyPrefix)
-	if err := runtime.Add(tokens.NewDefaultSigningKeyComponent(
-		zoneIngressSigningKeyManager,
-		log.WithValues("secretPrefix", zoneingress.ZoneIngressSigningKeyPrefix))); err != nil {
-		return err
+		zoneSigningKeyManager := tokens.NewSigningKeyManager(runtime.ResourceManager(), zone.SigningKeyPrefix)
+		if err := runtime.Add(tokens.NewDefaultSigningKeyComponent(
+			zoneSigningKeyManager,
+			log.WithValues("secretPrefix", zoneingress.ZoneIngressSigningKeyPrefix),
+		)); err != nil {
+			return err
+		}
+		if err := runtime.Add(defaultsComponent); err != nil {
+			return err
+		}
 	}
 
-	zoneSigningKeyManager := tokens.NewSigningKeyManager(runtime.ResourceManager(), zone.SigningKeyPrefix)
-	if err := runtime.Add(tokens.NewDefaultSigningKeyComponent(
-		zoneSigningKeyManager,
-		log.WithValues("secretPrefix", zoneingress.ZoneIngressSigningKeyPrefix),
-	)); err != nil {
-		return err
+	if runtime.Config().Mode != config_core.Global { // Envoy Admin CA is not synced in multizone and not needed in Global CP.
+		if err := runtime.Add(&EnvoyAdminCaDefaultComponent{ResManager: runtime.ResourceManager()}); err != nil {
+			return err
+		}
 	}
-
-	return runtime.Add(defaultsComponent)
+	return nil
 }
 
 func NewDefaultsComponent(config *kuma_cp.Defaults, cpMode config_core.CpMode, environment config_core.EnvironmentType, resManager core_manager.ResourceManager, resStore store.ResourceStore) component.Component {

pkg/defaults/envoy_admin_ca.go

@@ -0,0 +1,60 @@
+package defaults
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sethvargo/go-retry"
+
+	"github.com/kumahq/kuma/pkg/core/resources/manager"
+	"github.com/kumahq/kuma/pkg/core/resources/store"
+	"github.com/kumahq/kuma/pkg/core/runtime/component"
+	"github.com/kumahq/kuma/pkg/envoy/admin/tls"
+)
+
+type EnvoyAdminCaDefaultComponent struct {
+	ResManager manager.ResourceManager
+}
+
+var _ component.Component = &EnvoyAdminCaDefaultComponent{}
+
+func (e *EnvoyAdminCaDefaultComponent) Start(stop <-chan struct{}) error {
+	ctx, cancelFn := context.WithCancel(context.Background())
+	go func() {
+		<-stop
+		cancelFn()
+	}()
+	return retry.Do(ctx, retry.WithMaxDuration(10*time.Minute, retry.NewConstant(5*time.Second)), func(ctx context.Context) error {
+		if err := e.ensureEnvoyAdminCaExist(ctx); err != nil {
+			log.V(1).Info("could not ensure that Envoy Admin CA exists. Retrying.", "err", err)
+			return retry.RetryableError(err)
+		}
+		return nil
+	})
+}
+
+func (e EnvoyAdminCaDefaultComponent) NeedLeaderElection() bool {
+	return true
+}
+
+func (e *EnvoyAdminCaDefaultComponent) ensureEnvoyAdminCaExist(ctx context.Context) error {
+	_, err := tls.LoadCA(ctx, e.ResManager)
+	if err == nil {
+		log.V(1).Info("Envoy Admin CA already exists. Skip creating Envoy Admin CA.")
+		return nil
+	}
+	if !store.IsResourceNotFound(err) {
+		return errors.Wrap(err, "error while loading admin client certificate")
+	}
+	log.V(1).Info("trying to create Envoy Admin CA")
+	pair, err := tls.GenerateCA()
+	if err != nil {
+		return errors.Wrap(err, "could not generate admin client certificate")
+	}
+	if err := tls.CreateCA(ctx, *pair, e.ResManager); err != nil {
+		return errors.Wrap(err, "could not create admin client certificate")
+	}
+	log.Info("Envoy Admin CA created")
+	return nil
+}

pkg/defaults/envoy_admin_ca_test.go

@@ -0,0 +1,34 @@
+package defaults_test
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	core_system "github.com/kumahq/kuma/pkg/core/resources/apis/system"
+	core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
+	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
+	"github.com/kumahq/kuma/pkg/defaults"
+	envoy_admin_tls "github.com/kumahq/kuma/pkg/envoy/admin/tls"
+	resources_memory "github.com/kumahq/kuma/pkg/plugins/resources/memory"
+)
+
+var _ = Describe("Envoy Admin CA defaults", func() {
+	It("should create Envoy Admin CA", func() {
+		// given
+		store := resources_memory.NewStore()
+		manager := core_manager.NewResourceManager(store)
+		component := defaults.EnvoyAdminCaDefaultComponent{
+			ResManager: manager,
+		}
+
+		// when
+		err := component.Start(nil)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		err = manager.Get(context.Background(), core_system.NewGlobalSecretResource(), core_store.GetBy(envoy_admin_tls.GlobalSecretKey))
+		Expect(err).ToNot(HaveOccurred())
+	})
+})