xds_client_test.go

/*
 *
 * Copyright 2020 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package xds

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"strings"
	"testing"
	"time"

	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/tls/certprovider"
	"google.golang.org/grpc/internal"
	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
	"google.golang.org/grpc/internal/grpctest"
	"google.golang.org/grpc/internal/testutils"
	"google.golang.org/grpc/internal/xds"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/testdata"
)

const (
	defaultTestTimeout      = 10 * time.Second
	defaultTestShortTimeout = 10 * time.Millisecond
	defaultTestCertSAN      = "abc.test.example.com"
	authority               = "authority"
)

type s struct {
	grpctest.Tester
}

func Test(t *testing.T) {
	grpctest.RunSubTests(t, s{})
}

// Helper function to create a real TLS client credentials which is used as
// fallback credentials from multiple tests.
func makeFallbackClientCreds(t *testing.T) credentials.TransportCredentials {
	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
	if err != nil {
		t.Fatal(err)
	}
	return creds
}

// testServer is a no-op server which listens on a local TCP port for incoming
// connections, and performs a manual TLS handshake on the received raw
// connection using a user specified handshake function. It then makes the
// result of the handshake operation available through a channel for tests to
// inspect. Tests should stop the testServer as part of their cleanup.
type testServer struct {
	lis           net.Listener
	address       string             // Listening address of the test server.
	handshakeFunc testHandshakeFunc  // Test specified handshake function.
	hsResult      *testutils.Channel // Channel to deliver handshake results.
}

// handshakeResult wraps the result of the handshake operation on the test
// server. It consists of TLS connection state and an error, if the handshake
// failed. This result is delivered on the `hsResult` channel on the testServer.
type handshakeResult struct {
	connState tls.ConnectionState
	err       error
}

// Configurable handshake function for the testServer. Tests can set this to
// simulate different conditions like handshake success, failure, timeout etc.
type testHandshakeFunc func(net.Conn) handshakeResult

// newTestServerWithHandshakeFunc starts a new testServer which listens for
// connections on a local TCP port, and uses the provided custom handshake
// function to perform TLS handshake.
func newTestServerWithHandshakeFunc(f testHandshakeFunc) *testServer {
	ts := &testServer{
		handshakeFunc: f,
		hsResult:      testutils.NewChannel(),
	}
	ts.start()
	return ts
}

// starts actually starts listening on a local TCP port, and spawns a goroutine
// to handle new connections.
func (ts *testServer) start() error {
	lis, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return err
	}
	ts.lis = lis
	ts.address = lis.Addr().String()
	go ts.handleConn()
	return nil
}

// handleconn accepts a new raw connection, and invokes the test provided
// handshake function to perform TLS handshake, and returns the result on the
// `hsResult` channel.
func (ts *testServer) handleConn() {
	for {
		rawConn, err := ts.lis.Accept()
		if err != nil {
			// Once the listeners closed, Accept() will return with an error.
			return
		}
		hsr := ts.handshakeFunc(rawConn)
		ts.hsResult.Send(hsr)
	}
}

// stop closes the associated listener which causes the connection handling
// goroutine to exit.
func (ts *testServer) stop() {
	ts.lis.Close()
}

// A handshake function which simulates a successful handshake without client
// authentication (server does not request for client certificate during the
// handshake here).
func testServerTLSHandshake(rawConn net.Conn) handshakeResult {
	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
	if err != nil {
		return handshakeResult{err: err}
	}
	cfg := &tls.Config{Certificates: []tls.Certificate{cert}}
	conn := tls.Server(rawConn, cfg)
	if err := conn.Handshake(); err != nil {
		return handshakeResult{err: err}
	}
	return handshakeResult{connState: conn.ConnectionState()}
}

// A handshake function which simulates a successful handshake with mutual
// authentication.
func testServerMutualTLSHandshake(rawConn net.Conn) handshakeResult {
	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
	if err != nil {
		return handshakeResult{err: err}
	}
	pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
	if err != nil {
		return handshakeResult{err: err}
	}
	roots := x509.NewCertPool()
	roots.AppendCertsFromPEM(pemData)
	cfg := &tls.Config{
		Certificates: []tls.Certificate{cert},
		ClientCAs:    roots,
	}
	conn := tls.Server(rawConn, cfg)
	if err := conn.Handshake(); err != nil {
		return handshakeResult{err: err}
	}
	return handshakeResult{connState: conn.ConnectionState()}
}

// fakeProvider is an implementation of the certprovider.Provider interface
// which returns the configured key material and error in calls to
// KeyMaterial().
type fakeProvider struct {
	km  *certprovider.KeyMaterial
	err error
}

func (f *fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
	return f.km, f.err
}

func (f *fakeProvider) Close() {}

// makeIdentityProvider creates a new instance of the fakeProvider returning the
// identity key material specified in the provider file paths.
func makeIdentityProvider(t *testing.T, certPath, keyPath string) certprovider.Provider {
	t.Helper()
	cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath))
	if err != nil {
		t.Fatal(err)
	}
	return &fakeProvider{km: &certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}}
}

// makeRootProvider creates a new instance of the fakeProvider returning the
// root key material specified in the provider file paths.
func makeRootProvider(t *testing.T, caPath string) *fakeProvider {
	pemData, err := ioutil.ReadFile(testdata.Path(caPath))
	if err != nil {
		t.Fatal(err)
	}
	roots := x509.NewCertPool()
	roots.AppendCertsFromPEM(pemData)
	return &fakeProvider{km: &certprovider.KeyMaterial{Roots: roots}}
}

// newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo
// context value added to it.
func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sanExactMatch string) context.Context {
	// Creating the HandshakeInfo and adding it to the attributes is very
	// similar to what the CDS balancer would do when it intercepts calls to
	// NewSubConn().
	info := xdsinternal.NewHandshakeInfo(root, identity)
	if sanExactMatch != "" {
		info.SetSANMatchers([]xds.StringMatcher{{ExactMatch: newStringP(sanExactMatch)}})
	}
	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)

	// Moving the attributes from the resolver.Address to the context passed to
	// the handshaker is done in the transport layer. Since we directly call the
	// handshaker in these tests, we need to do the same here.
	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
	return contextWithHandshakeInfo(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
}

// compareAuthInfo compares the AuthInfo received on the client side after a
// successful handshake with the authInfo available on the testServer.
func compareAuthInfo(ctx context.Context, ts *testServer, ai credentials.AuthInfo) error {
	if ai.AuthType() != "tls" {
		return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls")
	}
	info, ok := ai.(credentials.TLSInfo)
	if !ok {
		return fmt.Errorf("ClientHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})
	}
	gotState := info.State

	// Read the handshake result from the testServer which contains the TLS
	// connection state and compare it with the one received on the client-side.
	val, err := ts.hsResult.Receive(ctx)
	if err != nil {
		return fmt.Errorf("testServer failed to return handshake result: %v", err)
	}
	hsr := val.(handshakeResult)
	if hsr.err != nil {
		return fmt.Errorf("testServer handshake failure: %v", hsr.err)
	}
	// AuthInfo contains a variety of information. We only verify a subset here.
	// This is the same subset which is verified in TLS credentials tests.
	if err := compareConnState(gotState, hsr.connState); err != nil {
		return err
	}
	return nil
}

func compareConnState(got, want tls.ConnectionState) error {
	switch {
	case got.Version != want.Version:
		return fmt.Errorf("TLS.ConnectionState got Version: %v, want: %v", got.Version, want.Version)
	case got.HandshakeComplete != want.HandshakeComplete:
		return fmt.Errorf("TLS.ConnectionState got HandshakeComplete: %v, want: %v", got.HandshakeComplete, want.HandshakeComplete)
	case got.CipherSuite != want.CipherSuite:
		return fmt.Errorf("TLS.ConnectionState got CipherSuite: %v, want: %v", got.CipherSuite, want.CipherSuite)
	case got.NegotiatedProtocol != want.NegotiatedProtocol:
		return fmt.Errorf("TLS.ConnectionState got NegotiatedProtocol: %v, want: %v", got.NegotiatedProtocol, want.NegotiatedProtocol)
	}
	return nil
}

// TestClientCredsWithoutFallback verifies that the call to
// NewClientCredentials() fails when no fallback is specified.
func (s) TestClientCredsWithoutFallback(t *testing.T) {
	if _, err := NewClientCredentials(ClientOptions{}); err == nil {
		t.Fatal("NewClientCredentials() succeeded without specifying fallback")
	}
}

// TestClientCredsInvalidHandshakeInfo verifies scenarios where the passed in
// HandshakeInfo is invalid because it does not contain the expected certificate
// providers.
func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) {
	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{}, "")
	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
		t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo")
	}
}

// TestClientCredsProviderFailure verifies the cases where an expected
// certificate provider is missing in the HandshakeInfo value in the context.
func (s) TestClientCredsProviderFailure(t *testing.T) {
	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	tests := []struct {
		desc             string
		rootProvider     certprovider.Provider
		identityProvider certprovider.Provider
		wantErr          string
	}{
		{
			desc:         "erroring root provider",
			rootProvider: &fakeProvider{err: errors.New("root provider error")},
			wantErr:      "root provider error",
		},
		{
			desc:             "erroring identity provider",
			rootProvider:     &fakeProvider{km: &certprovider.KeyMaterial{}},
			identityProvider: &fakeProvider{err: errors.New("identity provider error")},
			wantErr:          "identity provider error",
		},
	}
	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
			defer cancel()
			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider, "")
			if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) {
				t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
			}
		})
	}
}

// TestClientCredsSuccess verifies successful client handshake cases.
func (s) TestClientCredsSuccess(t *testing.T) {
	tests := []struct {
		desc             string
		handshakeFunc    testHandshakeFunc
		handshakeInfoCtx func(ctx context.Context) context.Context
	}{
		{
			desc:          "fallback",
			handshakeFunc: testServerTLSHandshake,
			handshakeInfoCtx: func(ctx context.Context) context.Context {
				// Since we don't add a HandshakeInfo to the context, the
				// ClientHandshake() method will delegate to the fallback.
				return ctx
			},
		},
		{
			desc:          "TLS",
			handshakeFunc: testServerTLSHandshake,
			handshakeInfoCtx: func(ctx context.Context) context.Context {
				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
			},
		},
		{
			desc:          "mTLS",
			handshakeFunc: testServerMutualTLSHandshake,
			handshakeInfoCtx: func(ctx context.Context) context.Context {
				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), defaultTestCertSAN)
			},
		},
		{
			desc:          "mTLS with no acceptedSANs specified",
			handshakeFunc: testServerMutualTLSHandshake,
			handshakeInfoCtx: func(ctx context.Context) context.Context {
				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), "")
			},
		},
	}

	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
			defer ts.stop()

			opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
			creds, err := NewClientCredentials(opts)
			if err != nil {
				t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
			}

			conn, err := net.Dial("tcp", ts.address)
			if err != nil {
				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
			}
			defer conn.Close()

			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
			defer cancel()
			_, ai, err := creds.ClientHandshake(test.handshakeInfoCtx(ctx), authority, conn)
			if err != nil {
				t.Fatalf("ClientHandshake() returned failed: %q", err)
			}
			if err := compareAuthInfo(ctx, ts, ai); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func (s) TestClientCredsHandshakeTimeout(t *testing.T) {
	clientDone := make(chan struct{})
	// A handshake function which simulates a handshake timeout from the
	// server-side by simply blocking on the client-side handshake to timeout
	// and not writing any handshake data.
	hErr := errors.New("server handshake error")
	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
		<-clientDone
		return handshakeResult{err: hErr}
	})
	defer ts.stop()

	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	conn, err := net.Dial("tcp", ts.address)
	if err != nil {
		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
	defer sCancel()
	ctx := newTestContextWithHandshakeInfo(sCtx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
		t.Fatal("ClientHandshake() succeeded when expected to timeout")
	}
	close(clientDone)

	// Read the handshake result from the testServer and make sure the expected
	// error is returned.
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	val, err := ts.hsResult.Receive(ctx)
	if err != nil {
		t.Fatalf("testServer failed to return handshake result: %v", err)
	}
	hsr := val.(handshakeResult)
	if hsr.err != hErr {
		t.Fatalf("testServer handshake returned error: %v, want: %v", hsr.err, hErr)
	}
}

// TestClientCredsHandshakeFailure verifies different handshake failure cases.
func (s) TestClientCredsHandshakeFailure(t *testing.T) {
	tests := []struct {
		desc          string
		handshakeFunc testHandshakeFunc
		rootProvider  certprovider.Provider
		san           string
		wantErr       string
	}{
		{
			desc:          "cert validation failure",
			handshakeFunc: testServerTLSHandshake,
			rootProvider:  makeRootProvider(t, "x509/client_ca_cert.pem"),
			san:           defaultTestCertSAN,
			wantErr:       "x509: certificate signed by unknown authority",
		},
		{
			desc:          "SAN mismatch",
			handshakeFunc: testServerTLSHandshake,
			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
			san:           "bad-san",
			wantErr:       "does not match any of the accepted SANs",
		},
	}

	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
			defer ts.stop()

			opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
			creds, err := NewClientCredentials(opts)
			if err != nil {
				t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
			}

			conn, err := net.Dial("tcp", ts.address)
			if err != nil {
				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
			}
			defer conn.Close()

			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
			defer cancel()
			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, nil, test.san)
			if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil || !strings.Contains(err.Error(), test.wantErr) {
				t.Fatalf("ClientHandshake() returned %q, wantErr %q", err, test.wantErr)
			}
		})
	}
}

// TestClientCredsProviderSwitch verifies the case where the first attempt of
// ClientHandshake fails because of a handshake failure. Then we update the
// certificate provider and the second attempt succeeds. This is an
// approximation of the flow of events when the control plane specifies new
// security config which results in new certificate providers being used.
func (s) TestClientCredsProviderSwitch(t *testing.T) {
	ts := newTestServerWithHandshakeFunc(testServerTLSHandshake)
	defer ts.stop()

	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	conn, err := net.Dial("tcp", ts.address)
	if err != nil {
		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	// Create a root provider which will fail the handshake because it does not
	// use the correct trust roots.
	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil)
	handshakeInfo.SetSANMatchers([]xds.StringMatcher{{ExactMatch: newStringP(defaultTestCertSAN)}})

	// We need to repeat most of what newTestContextWithHandshakeInfo() does
	// here because we need access to the underlying HandshakeInfo so that we
	// can update it before the next call to ClientHandshake().
	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, handshakeInfo)
	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
	ctx = contextWithHandshakeInfo(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
		t.Fatal("ClientHandshake() succeeded when expected to fail")
	}
	// Drain the result channel on the test server so that we can inspect the
	// result for the next handshake.
	_, err = ts.hsResult.Receive(ctx)
	if err != nil {
		t.Errorf("testServer failed to return handshake result: %v", err)
	}

	conn, err = net.Dial("tcp", ts.address)
	if err != nil {
		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	// Create a new root provider which uses the correct trust roots. And update
	// the HandshakeInfo with the new provider.
	root2 := makeRootProvider(t, "x509/server_ca_cert.pem")
	handshakeInfo.SetRootCertProvider(root2)
	_, ai, err := creds.ClientHandshake(ctx, authority, conn)
	if err != nil {
		t.Fatalf("ClientHandshake() returned failed: %q", err)
	}
	if err := compareAuthInfo(ctx, ts, ai); err != nil {
		t.Fatal(err)
	}
}

// TestClientClone verifies the Clone() method on client credentials.
func (s) TestClientClone(t *testing.T) {
	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	orig, err := NewClientCredentials(opts)
	if err != nil {
		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	// The credsImpl does not have any exported fields, and it does not make
	// sense to use any cmp options to look deep into. So, all we make sure here
	// is that the cloned object points to a different location in memory.
	if clone := orig.Clone(); clone == orig {
		t.Fatal("return value from Clone() doesn't point to new credentials instance")
	}
}

func newStringP(s string) *string {
	return &s
}