aeraki and envoy communicate through mTLS (#401)

Signed-off-by: wangshuide <wangshuide@huawei.com>
aeraki-mesh · Dec 19, 2023 · 9cf99cd · 9cf99cd
1 parent 6271691
commit 9cf99cd
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 2 deletions.
diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go
@@ -18,10 +18,12 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 
 	//nolint
 	_ "net/http/pprof" // pprof
@@ -163,6 +165,9 @@ func NewServer(args *AerakiArgs) (*Server, error) {
 	if err := server.initRootCA(); err != nil {
 		return nil, fmt.Errorf("error initializing root ca: %v", err)
 	}
+	if err := server.initXdsServer(); err != nil {
+		return nil, fmt.Errorf("error initializing xds server: %v", err)
+	}
 	if err := server.initSecureWebhookServer(args); err != nil {
 		return nil, fmt.Errorf("error initializing webhook server: %v", err)
 	}
@@ -177,6 +182,25 @@ func NewServer(args *AerakiArgs) (*Server, error) {
 	return server, err
 }
 
+func (s *Server) initXdsServer() error {
+	pool := x509.NewCertPool()
+	istiodCACertPath := "/var/run/secrets/istio/root-cert.pem"
+	caCrt, err := os.ReadFile(istiodCACertPath)
+	if err != nil {
+		return fmt.Errorf("failed to read istio ca cert file: %v", err)
+	}
+	pool.AppendCertsFromPEM(caCrt)
+
+	s.xdsServer.TLSConfig = tls.Config{
+		GetCertificate: s.getAerakiCertificate,
+		MinVersion:     tls.VersionTLS12,
+		ClientAuth:     tls.RequireAndVerifyClientCert,
+		ClientCAs:      pool,
+	}
+
+	return nil
+}
+
 func (s *Server) initAerakiServer(args *AerakiArgs) {
 	// make sure we have a readiness probe before serving HTTP to avoid marking ready too soon
 	s.initReadinessProbes()

diff --git a/internal/controller/kube/sidecar_bootstrap_config.go b/internal/controller/kube/sidecar_bootstrap_config.go
@@ -72,6 +72,48 @@ var bootstrapConfig = `
                          ]
                       }
                    ]
+                },
+                "transport_socket": {
+                   "name": "envoy.transport_sockets.tls",
+                   "typed_config": {
+                      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
+                      "common_tls_context": {
+                         "validation_context_sds_secret_config": {
+                            "name": "ROOTCA",
+                            "sds_config": {
+                               "api_config_source": {
+                                  "api_type": "GRPC",
+                                  "grpc_services": [{
+                                     "envoy_grpc": {
+                                        "cluster_name": "sds-grpc"
+                                     }
+                                  }],
+                                  "set_node_on_first_message_only": true,
+                                  "transport_api_version": "V3"
+                               },
+                               "initial_fetch_timeout": "0s",
+                               "resource_api_version": "V3"
+                            }
+                         },
+                         "tls_certificate_sds_secret_configs": [{
+                            "name": "default",
+                            "sds_config": {
+                               "api_config_source": {
+                                  "api_type": "GRPC",
+                                  "grpc_services": [{
+                                     "envoy_grpc": {
+                                        "cluster_name": "sds-grpc"
+                                     }
+                                  }],
+                                  "set_node_on_first_message_only": true,
+                                  "transport_api_version": "V3"
+                               },
+                               "initial_fetch_timeout": "0s",
+                               "resource_api_version": "V3"
+                            }
+                         }]
+                      }
+                   }
                 }
              }
           ]

diff --git a/internal/xds/server.go b/internal/xds/server.go
@@ -16,8 +16,11 @@ package xds
 
 import (
 	"context"
+	"crypto/tls"
 	"net"
 
+	"google.golang.org/grpc/credentials"
+
 	routeservice "github.com/envoyproxy/go-control-plane/envoy/service/route/v3"
 	cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
 	serverv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3"
@@ -40,8 +43,9 @@ type cacheMgr interface {
 
 // Server serves xDS resources to Envoy sidecars
 type Server struct {
-	addr     string
-	cacheMgr cacheMgr
+	addr      string
+	cacheMgr  cacheMgr
+	TLSConfig tls.Config
 }
 
 // NewServer creates a xDS server
@@ -60,6 +64,8 @@ func (s *Server) Run(stopCh <-chan struct{}) {
 	// availability problems.
 	var grpcOptions []grpc.ServerOption
 	grpcOptions = append(grpcOptions, grpc.MaxConcurrentStreams(grpcMaxConcurrentStreams))
+	tlsCreds := credentials.NewTLS(&s.TLSConfig)
+	grpcOptions = append(grpcOptions, grpc.Creds(tlsCreds))
 	grpcServer := grpc.NewServer(grpcOptions...)
 
 	lis, err := net.Listen("tcp", s.addr)

diff --git a/k8s/aeraki-bootstrap-config.yaml b/k8s/aeraki-bootstrap-config.yaml
@@ -69,6 +69,48 @@ data:
                          ]
                       }
                    ]
+                },
+                "transport_socket": {
+                   "name": "envoy.transport_sockets.tls",
+                   "typed_config": {
+                      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
+                      "common_tls_context": {
+                         "validation_context_sds_secret_config": {
+                            "name": "ROOTCA",
+                            "sds_config": {
+                               "api_config_source": {
+                                  "api_type": "GRPC",
+                                  "grpc_services": [{
+                                     "envoy_grpc": {
+                                        "cluster_name": "sds-grpc"
+                                     }
+                                  }],
+                                  "set_node_on_first_message_only": true,
+                                  "transport_api_version": "V3"
+                               },
+                               "initial_fetch_timeout": "0s",
+                               "resource_api_version": "V3"
+                            }
+                         },
+                         "tls_certificate_sds_secret_configs": [{
+                            "name": "default",
+                            "sds_config": {
+                               "api_config_source": {
+                                  "api_type": "GRPC",
+                                  "grpc_services": [{
+                                     "envoy_grpc": {
+                                        "cluster_name": "sds-grpc"
+                                     }
+                                  }],
+                                  "set_node_on_first_message_only": true,
+                                  "transport_api_version": "V3"
+                               },
+                               "initial_fetch_timeout": "0s",
+                               "resource_api_version": "V3"
+                            }
+                         }]
+                      }
+                   }
                 }
              }
           ]