support SDS server

zhenhli-microsoft · Oct 11, 2021 · 6a583fc · 6a583fc
1 parent f362f4d
commit 6a583fc
Show file tree

Hide file tree

Showing 8 changed files with 154 additions and 80 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 ARG BUILDPLATFORM=linux/amd64
-ARG BUILD_BASE_IMAGE
+ARG BUILD_BASE_IMAGE=golang:1.17.0
 
 FROM --platform=$BUILDPLATFORM $BUILD_BASE_IMAGE AS build
 WORKDIR /contour

diff --git a/apis/projectcontour/v1/httpproxy.go b/apis/projectcontour/v1/httpproxy.go
@@ -226,11 +226,15 @@ type VirtualHost struct {
 // TLS describes tls properties. The SNI names that will be matched on
 // are described in the HTTPProxy's Spec.VirtualHost.Fqdn field.
 type TLS struct {
-	// SecretName is the name of a TLS secret in the current namespace.
+	// SecretName is the name of a TLS secret.
+	// If SDS is not enabled, the secret must be in the current namespace.
+	// If SDS is enabled, the certificate will be fetched from SDS server.
 	// Either SecretName or Passthrough must be specified, but not both.
 	// If specified, the named secret must contain a matching certificate
 	// for the virtual host's FQDN.
 	SecretName string `json:"secretName,omitempty"`
+	// Whether fetch certificate from SDS server or kubernetes secret resoruce
+	EnableSDS bool `json:"enableSDS,omitempty"`
 	// MinimumProtocolVersion is the minimum TLS version this vhost should
 	// negotiate. Valid options are `1.2` (default) and `1.3`. Any other value
 	// defaults to TLS 1.2.

diff --git a/cmd/contour/bootstrap.go b/cmd/contour/bootstrap.go
@@ -30,11 +30,14 @@ func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.Boo
 	bootstrap.Flag("admin-port", "DEPRECATED: Envoy admin interface port.").IntVar(&config.AdminPort)
 	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress)
 	bootstrap.Flag("xds-port", "xDS gRPC API port.").IntVar(&config.XDSGRPCPort)
+	bootstrap.Flag("sds-address", "sDS gRPC API address.").StringVar(&config.SDSAddress)
+	bootstrap.Flag("sds-port", "sDS gRPC API port.").IntVar(&config.SDSGRPCPort)
 	bootstrap.Flag("envoy-cafile", "CA Filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_CAFILE").StringVar(&config.GrpcCABundle)
 	bootstrap.Flag("envoy-cert-file", "Client certificate filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
 	bootstrap.Flag("envoy-key-file", "Client key filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
 	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace)
 	bootstrap.Flag("xds-resource-version", "The versions of the xDS resources to request from Contour.").Default("v3").StringVar((*string)(&config.XDSResourceVersion))
+	bootstrap.Flag("sds-resource-version", "The versions of the sDS resources to request from Contour.").Default("v3").StringVar((*string)(&config.SDSResourceVersion))
 	bootstrap.Flag("dns-lookup-family", "Defines what DNS Resolution Policy to use for Envoy -> Contour cluster name lookup. Either v4, v6 or auto.").StringVar(&config.DNSLookupFamily)
 	return bootstrap, &config
 }
diff --git a/internal/dag/dag.go b/internal/dag/dag.go
@@ -813,10 +813,11 @@ func (s *ServiceCluster) Rebalance() {
 // Secret represents a K8s Secret for TLS usage as a DAG Vertex. A Secret is
 // a leaf in the DAG.
 type Secret struct {
-	Object *v1.Secret
+	Object        *v1.Secret
+	SdsSecretName string
 }
 
-func (s *Secret) Name() string       { return s.Object.Name }
+func (s *Secret) Name() string       { return stringOrDefault(s.Object.Name, s.SdsSecretName) }
 func (s *Secret) Namespace() string  { return s.Object.Namespace }
 func (s *Secret) Visit(func(Vertex)) {}
 

diff --git a/internal/dag/httpproxy_processor.go b/internal/dag/httpproxy_processor.go
@@ -181,22 +181,28 @@ func (p *HTTPProxyProcessor) computeHTTPProxy(proxy *contour_api_v1.HTTPProxy) {
 
 		// Attach secrets to TLS enabled vhosts.
 		if !tls.Passthrough {
-			secretName := k8s.NamespacedNameFrom(tls.SecretName, k8s.DefaultNamespace(proxy.Namespace))
-			sec, err := p.source.LookupSecret(secretName, validSecret)
-			if err != nil {
-				validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "SecretNotValid",
-					"Spec.VirtualHost.TLS Secret %q is invalid: %s", tls.SecretName, err)
-				return
-			}
+			svhost := p.dag.EnsureSecureVirtualHost(ListenerName{Name: host, ListenerName: "ingress_https"})
+			if !tls.EnableSDS {
+				secretName := k8s.NamespacedNameFrom(tls.SecretName, k8s.DefaultNamespace(proxy.Namespace))
+				sec, err := p.source.LookupSecret(secretName, validSecret)
+				if err != nil {
+					validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "SecretNotValid",
+						"Spec.VirtualHost.TLS Secret %q is invalid: %s", tls.SecretName, err)
+					return
+				}
 
-			if !p.source.DelegationPermitted(secretName, proxy.Namespace) {
-				validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "DelegationNotPermitted",
-					"Spec.VirtualHost.TLS Secret %q certificate delegation not permitted", tls.SecretName)
-				return
+				if !p.source.DelegationPermitted(secretName, proxy.Namespace) {
+					validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "DelegationNotPermitted",
+						"Spec.VirtualHost.TLS Secret %q certificate delegation not permitted", tls.SecretName)
+					return
+				}
+				svhost.Secret = sec
+			} else {
+				svhost.Secret = &Secret{
+					SdsSecretName: tls.SecretName,
+				}
 			}
 
-			svhost := p.dag.EnsureSecureVirtualHost(ListenerName{Name: host, ListenerName: "ingress_https"})
-			svhost.Secret = sec
 			// default to a minimum TLS version of 1.2 if it's not specified
 			svhost.MinTLSVersion = annotation.MinTLSVersion(tls.MinimumProtocolVersion, "1.2")
 
@@ -226,20 +232,26 @@ func (p *HTTPProxyProcessor) computeHTTPProxy(proxy *contour_api_v1.HTTPProxy) {
 					return
 				}
 
-				sec, err = p.source.LookupSecret(*p.FallbackCertificate, validSecret)
-				if err != nil {
-					validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "FallbackNotValid",
-						"Spec.Virtualhost.TLS Secret %q fallback certificate is invalid: %s", p.FallbackCertificate, err)
-					return
-				}
+				if !tls.EnableSDS {
+					sec, err := p.source.LookupSecret(*p.FallbackCertificate, validSecret)
+					if err != nil {
+						validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "FallbackNotValid",
+							"Spec.Virtualhost.TLS Secret %q fallback certificate is invalid: %s", p.FallbackCertificate, err)
+						return
+					}
 
-				if !p.source.DelegationPermitted(*p.FallbackCertificate, proxy.Namespace) {
-					validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "FallbackNotDelegated",
-						"Spec.VirtualHost.TLS fallback Secret %q is not configured for certificate delegation", p.FallbackCertificate)
-					return
-				}
+					if !p.source.DelegationPermitted(*p.FallbackCertificate, proxy.Namespace) {
+						validCond.AddErrorf(contour_api_v1.ConditionTypeTLSError, "FallbackNotDelegated",
+							"Spec.VirtualHost.TLS fallback Secret %q is not configured for certificate delegation", p.FallbackCertificate)
+						return
+					}
+					svhost.FallbackCertificate = sec
+				} else {
+					svhost.FallbackCertificate = &Secret{
+						SdsSecretName: p.FallbackCertificate.Name,
+					}
 
-				svhost.FallbackCertificate = sec
+				}
 			}
 
 			// Fill in DownstreamValidation when external client validation is enabled.

diff --git a/internal/envoy/bootstrap.go b/internal/envoy/bootstrap.go
@@ -62,6 +62,18 @@ type BootstrapConfig struct {
 	// Defaults to "v3"
 	XDSResourceVersion config.ResourceVersion
 
+	// SDSAddress is the TCP address of the gRPC SDS server.
+	// Defaults to "" which means SDS is disabled.
+	SDSAddress string
+
+	// SDSGRPCPort is the management server port that provides the v3 gRPC API.
+	// Defaults to 8234.
+	SDSGRPCPort int
+
+	// SDSResourceVersion defines the SDS Server Version to use.
+	// Defaults to "v3"
+	SDSResourceVersion config.ResourceVersion
+
 	// Namespace is the namespace where Contour is running
 	Namespace string
 
@@ -94,6 +106,8 @@ type BootstrapConfig struct {
 
 func (c *BootstrapConfig) GetXdsAddress() string { return stringOrDefault(c.XDSAddress, "127.0.0.1") }
 func (c *BootstrapConfig) GetXdsGRPCPort() int   { return intOrDefault(c.XDSGRPCPort, 8001) }
+func (c *BootstrapConfig) GetSdsAddress() string { return stringOrDefault(c.SDSAddress, "") }
+func (c *BootstrapConfig) GetSdsGRPCPort() int   { return intOrDefault(c.SDSGRPCPort, 8234) }
 func (c *BootstrapConfig) GetAdminAddress() string {
 	return stringOrDefault(c.AdminAddress, "/admin/admin.sock")
 }

diff --git a/internal/envoy/v3/auth.go b/internal/envoy/v3/auth.go
@@ -93,6 +93,28 @@ func validationContext(ca []byte, subjectName string, skipVerifyPeerCert bool) *
 
 // DownstreamTLSContext creates a new DownstreamTlsContext.
 func DownstreamTLSContext(serverSecret *dag.Secret, tlsMinProtoVersion envoy_v3_tls.TlsParameters_TlsProtocol, cipherSuites []string, peerValidationContext *dag.PeerValidationContext, alpnProtos ...string) *envoy_v3_tls.DownstreamTlsContext {
+	var configSource *envoy_api_v3_core.ConfigSource
+	if serverSecret.Object != nil {
+		configSource = ConfigSource("contour")
+	} else {
+		configSource = &envoy_api_v3_core.ConfigSource{
+			ResourceApiVersion: envoy_api_v3_core.ApiVersion_V3, //todo
+			ConfigSourceSpecifier: &envoy_api_v3_core.ConfigSource_ApiConfigSource{
+				&envoy_api_v3_core.ApiConfigSource{
+					ApiType:             envoy_api_v3_core.ApiConfigSource_GRPC,
+					TransportApiVersion: envoy_api_v3_core.ApiVersion_V3, //todo
+					GrpcServices: []*envoy_api_v3_core.GrpcService{{
+						TargetSpecifier: &envoy_api_v3_core.GrpcService_EnvoyGrpc_{
+							EnvoyGrpc: &envoy_api_v3_core.GrpcService_EnvoyGrpc{
+								ClusterName: "sds_server",
+							},
+						},
+					}},
+				},
+			},
+		}
+	}
+
 	context := &envoy_v3_tls.DownstreamTlsContext{
 		CommonTlsContext: &envoy_v3_tls.CommonTlsContext{
 			TlsParams: &envoy_v3_tls.TlsParameters{
@@ -101,8 +123,9 @@ func DownstreamTLSContext(serverSecret *dag.Secret, tlsMinProtoVersion envoy_v3_
 				CipherSuites:              cipherSuites,
 			},
 			TlsCertificateSdsSecretConfigs: []*envoy_v3_tls.SdsSecretConfig{{
-				Name:      envoy.Secretname(serverSecret),
-				SdsConfig: ConfigSource("contour"),
+				Name: envoy.Secretname(serverSecret),
+				// SdsConfig: ConfigSource("contour"),
+				SdsConfig: configSource,
 			}},
 			AlpnProtocols: alpnProtos,
 		},

diff --git a/internal/envoy/v3/bootstrap.go b/internal/envoy/v3/bootstrap.go
@@ -152,61 +152,78 @@ func bootstrap(c *envoy.BootstrapConfig) ([]bootstrapf, error) {
 }
 
 func bootstrapConfig(c *envoy.BootstrapConfig) *envoy_bootstrap_v3.Bootstrap {
+	var clusters []*envoy_cluster_v3.Cluster
+	clusters = []*envoy_cluster_v3.Cluster{{
+		DnsLookupFamily:      parseDNSLookupFamily(c.DNSLookupFamily),
+		Name:                 "contour",
+		AltStatName:          strings.Join([]string{c.Namespace, "contour", strconv.Itoa(c.GetXdsGRPCPort())}, "_"),
+		ConnectTimeout:       protobuf.Duration(5 * time.Second),
+		ClusterDiscoveryType: ClusterDiscoveryTypeForAddress(c.GetXdsAddress(), envoy_cluster_v3.Cluster_STRICT_DNS),
+		LbPolicy:             envoy_cluster_v3.Cluster_ROUND_ROBIN,
+		LoadAssignment: &envoy_endpoint_v3.ClusterLoadAssignment{
+			ClusterName: "contour",
+			Endpoints: Endpoints(
+				SocketAddress(c.GetXdsAddress(), c.GetXdsGRPCPort()),
+			),
+		},
+		UpstreamConnectionOptions: &envoy_cluster_v3.UpstreamConnectionOptions{
+			TcpKeepalive: &envoy_core_v3.TcpKeepalive{
+				KeepaliveProbes:   protobuf.UInt32(3),
+				KeepaliveTime:     protobuf.UInt32(30),
+				KeepaliveInterval: protobuf.UInt32(5),
+			},
+		},
+		TypedExtensionProtocolOptions: http2ProtocolOptions(),
+		CircuitBreakers: &envoy_cluster_v3.CircuitBreakers{
+			Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{{
+				Priority:           envoy_core_v3.RoutingPriority_HIGH,
+				MaxConnections:     protobuf.UInt32(100000),
+				MaxPendingRequests: protobuf.UInt32(100000),
+				MaxRequests:        protobuf.UInt32(60000000),
+				MaxRetries:         protobuf.UInt32(50),
+			}, {
+				Priority:           envoy_core_v3.RoutingPriority_DEFAULT,
+				MaxConnections:     protobuf.UInt32(100000),
+				MaxPendingRequests: protobuf.UInt32(100000),
+				MaxRequests:        protobuf.UInt32(60000000),
+				MaxRetries:         protobuf.UInt32(50),
+			}},
+		},
+	}, {
+		Name:                 "service-stats",
+		AltStatName:          strings.Join([]string{c.Namespace, "service-stats", strconv.Itoa(c.GetAdminPort())}, "_"),
+		ConnectTimeout:       protobuf.Duration(250 * time.Millisecond),
+		ClusterDiscoveryType: ClusterDiscoveryTypeForAddress(c.GetAdminAddress(), envoy_cluster_v3.Cluster_STATIC),
+		LbPolicy:             envoy_cluster_v3.Cluster_ROUND_ROBIN,
+		LoadAssignment: &envoy_endpoint_v3.ClusterLoadAssignment{
+			ClusterName: "service-stats",
+			Endpoints: Endpoints(
+				UnixSocketAddress(c.GetAdminAddress(), c.GetAdminPort()),
+			),
+		},
+	}}
+	if c.GetSdsAddress() != "" {
+		clusters = append(clusters, &envoy_cluster_v3.Cluster{
+			DnsLookupFamily:               parseDNSLookupFamily(c.DNSLookupFamily),
+			Name:                          "sds_server",
+			AltStatName:                   strings.Join([]string{c.Namespace, "sds_server", strconv.Itoa(c.GetSdsGRPCPort())}, "_"),
+			ConnectTimeout:                protobuf.Duration(5 * time.Second),
+			TypedExtensionProtocolOptions: http2ProtocolOptions(),
+			LoadAssignment: &envoy_endpoint_v3.ClusterLoadAssignment{
+				ClusterName: "sds_server",
+				Endpoints: Endpoints(
+					SocketAddress(c.GetSdsAddress(), c.GetSdsGRPCPort()),
+				),
+			},
+		})
+	}
 	return &envoy_bootstrap_v3.Bootstrap{
 		DynamicResources: &envoy_bootstrap_v3.Bootstrap_DynamicResources{
 			LdsConfig: ConfigSource("contour"),
 			CdsConfig: ConfigSource("contour"),
 		},
 		StaticResources: &envoy_bootstrap_v3.Bootstrap_StaticResources{
-			Clusters: []*envoy_cluster_v3.Cluster{{
-				DnsLookupFamily:      parseDNSLookupFamily(c.DNSLookupFamily),
-				Name:                 "contour",
-				AltStatName:          strings.Join([]string{c.Namespace, "contour", strconv.Itoa(c.GetXdsGRPCPort())}, "_"),
-				ConnectTimeout:       protobuf.Duration(5 * time.Second),
-				ClusterDiscoveryType: ClusterDiscoveryTypeForAddress(c.GetXdsAddress(), envoy_cluster_v3.Cluster_STRICT_DNS),
-				LbPolicy:             envoy_cluster_v3.Cluster_ROUND_ROBIN,
-				LoadAssignment: &envoy_endpoint_v3.ClusterLoadAssignment{
-					ClusterName: "contour",
-					Endpoints: Endpoints(
-						SocketAddress(c.GetXdsAddress(), c.GetXdsGRPCPort()),
-					),
-				},
-				UpstreamConnectionOptions: &envoy_cluster_v3.UpstreamConnectionOptions{
-					TcpKeepalive: &envoy_core_v3.TcpKeepalive{
-						KeepaliveProbes:   protobuf.UInt32(3),
-						KeepaliveTime:     protobuf.UInt32(30),
-						KeepaliveInterval: protobuf.UInt32(5),
-					},
-				},
-				TypedExtensionProtocolOptions: http2ProtocolOptions(),
-				CircuitBreakers: &envoy_cluster_v3.CircuitBreakers{
-					Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{{
-						Priority:           envoy_core_v3.RoutingPriority_HIGH,
-						MaxConnections:     protobuf.UInt32(100000),
-						MaxPendingRequests: protobuf.UInt32(100000),
-						MaxRequests:        protobuf.UInt32(60000000),
-						MaxRetries:         protobuf.UInt32(50),
-					}, {
-						Priority:           envoy_core_v3.RoutingPriority_DEFAULT,
-						MaxConnections:     protobuf.UInt32(100000),
-						MaxPendingRequests: protobuf.UInt32(100000),
-						MaxRequests:        protobuf.UInt32(60000000),
-						MaxRetries:         protobuf.UInt32(50),
-					}},
-				},
-			}, {
-				Name:                 "service-stats",
-				AltStatName:          strings.Join([]string{c.Namespace, "service-stats", strconv.Itoa(c.GetAdminPort())}, "_"),
-				ConnectTimeout:       protobuf.Duration(250 * time.Millisecond),
-				ClusterDiscoveryType: ClusterDiscoveryTypeForAddress(c.GetAdminAddress(), envoy_cluster_v3.Cluster_STATIC),
-				LbPolicy:             envoy_cluster_v3.Cluster_ROUND_ROBIN,
-				LoadAssignment: &envoy_endpoint_v3.ClusterLoadAssignment{
-					ClusterName: "service-stats",
-					Endpoints: Endpoints(
-						UnixSocketAddress(c.GetAdminAddress(), c.GetAdminPort()),
-					),
-				},
-			}},
+			Clusters: clusters,
 		},
 		Admin: &envoy_bootstrap_v3.Admin{
 			AccessLogPath: c.GetAdminAccessLogPath(),