From 6d26f6fa5afbd3561978cc5086076885a17deff7 Mon Sep 17 00:00:00 2001 From: Easwar Swaminathan Date: Tue, 12 Dec 2023 20:48:29 +0000 Subject: [PATCH] e2e style tests for client side --- internal/testutils/xds/e2e/setup_certs.go | 26 +- .../xds_client_certificate_providers_test.go | 362 ++++++++++++++++++ 2 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 test/xds/xds_client_certificate_providers_test.go diff --git a/internal/testutils/xds/e2e/setup_certs.go b/internal/testutils/xds/e2e/setup_certs.go index 799e1856487..dea39216259 100644 --- a/internal/testutils/xds/e2e/setup_certs.go +++ b/internal/testutils/xds/e2e/setup_certs.go @@ -87,7 +87,7 @@ func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials { } roots := x509.NewCertPool() if !roots.AppendCertsFromPEM(b) { - t.Fatal("failed to append certificates") + t.Fatal("Failed to append certificates") } return credentials.NewTLS(&tls.Config{ Certificates: []tls.Certificate{cert}, @@ -95,3 +95,27 @@ func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials { ServerName: "x.test.example.com", }) } + +// CreateServerTLSCredentials creates server-side TLS transport credentials +// using certificate and key files from testdata/x509 directory. +func CreateServerTLSCredentials(t *testing.T) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/server1_cert.pem, x509/server1_key.pem) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/client_ca_cert.pem) failed: %v", err) + } + ca := x509.NewCertPool() + if !ca.AppendCertsFromPEM(b) { + t.Fatal("Failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: ca, + }) +} diff --git a/test/xds/xds_client_certificate_providers_test.go b/test/xds/xds_client_certificate_providers_test.go new file mode 100644 index 00000000000..a2979ca1bea --- /dev/null +++ b/test/xds/xds_client_certificate_providers_test.go @@ -0,0 +1,362 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Tests the case where the bootstrap configuration contains no certificate +// providers, and xDS credentials with an insecure fallback is specified at dial +// time. The management server is configured to return client side xDS resources +// with no security configuration. The test verifies that the gRPC client is +// able to make RPCs to the backend which is configured to accept plaintext +// connections. This ensures that the insecure fallback credentials are getting +// used on the client. +func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to start management server: %v", err) + } + defer mgmtServer.Stop() + + // Create bootstrap configuration with no certificate providers. + nodeID := uuid.New().String() + bs, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS resolver with the above bootstrap configuration. + newResolver := internal.NewXDSResolverWithConfigForTesting + if newResolver == nil { + t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset") + } + resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Spin up a test backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure client side xDS resources on the management server, with no + // security configuration in the Cluster resource. + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the case where the bootstrap configuration contains no certificate +// providers, and xDS credentials with an insecure fallback is specified at dial +// time. The management server is configured to return client side xDS resources +// with an mTLS security configuration. The test verifies that the gRPC client +// moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and +// string. This ensures that when the certificate provider instance name +// specified in the security configuration is not present in the bootstrap, +// channel creation does not fail, but it moves to TRANSIENT_FAILURE and +// subsequent rpcs fail. +func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to start management server: %v", err) + } + defer mgmtServer.Stop() + + // Create bootstrap configuration with no certificate providers. + nodeID := uuid.New().String() + bs, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS resolver with the above bootstrap configuration. + newResolver := internal.NewXDSResolverWithConfigForTesting + if newResolver == nil { + t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset") + } + resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Spin up a test backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure client side xDS resources on the management server, with mTLS + // security configuration in the Cluster resource. + const serviceName = "my-service-client-side-xds" + const clusterName = "cluster-" + serviceName + const endpointsName = "endpoints-" + serviceName + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Make an RPC and ensure that expected error is returned. + wantErr := fmt.Sprintf("identitiy certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance) + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} + +// Tests the case where the bootstrap configuration contains one certificate +// provider, and xDS credentials with an insecure fallback is specified at dial +// time. The management server responds with three clusters: +// 1. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap +// 2. contains no security configuration, hence should use insecure fallback +// 3. contains invalid security configuration pointing to a non-existent +// certificate provider instance +// +// The test verifies that RPCs to the first two clusters succeed, while RPCs to +// the third cluster fails with an appropriate code and error message. +func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) { + // Spin up an xDS management server. This uses a bootstrap config with a + // certificate provider instance name e2e.ClientSideCertProviderInstance. + mgmtServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup() + + // Create test backends for all three clusters + // backend1 configured with TLS creds, represents cluster1 + // backend2 configured with insecure creds, represents cluster2 + // backend3 configured with insecure creds, represents cluster3 + creds := e2e.CreateServerTLSCredentials(t) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(creds)) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil) + defer server2.Stop() + server3 := stubserver.StartTestService(t, nil) + defer server3.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const clusterName3 = "cluster3-" + serviceName + const endpointsName1 = "endpoints1-" + serviceName + const endpointsName2 = "endpoints2-" + serviceName + const endpointsName3 = "endpoints3-" + serviceName + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + // Route configuration: + // - "/grpc.testing.TestService/EmptyCall" --> cluster1 + // - "/grpc.testing.TestService/UnaryCall" --> cluster2 + // - "/grpc.testing.TestService/FullDuplexCall" --> cluster3 + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3}, + }}, + }, + }, + }}, + }} + // Clusters: + // - cluster1 with cert provider name e2e.ClientSideCertProviderInstance. + // - cluster2 with no security configuration. + // - cluster3 with non-existent cert provider name. + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS), + e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone), + func() *v3clusterpb.Cluster { + cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS) + cluster3.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + }), + }, + } + return cluster3 + }(), + } + // Endpoints for each of the above clusters with backends created earlier. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make an RPC to be routed to cluster1 and verify that it succeeds. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server1.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + + } + + // Make an RPC to be routed to cluster2 and verify that it succeeds. + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server2.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + + // Make an RPC to be routed to cluster3 and verify that it fails. + const wantErr = `identitiy certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration` + if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +}