Skip to content

Commit

Permalink
Require Connect and TLS to generate peering tokens
Browse files Browse the repository at this point in the history 
By requiring Connect and a gRPC TLS listener we can automatically
configure TLS for all peering control-plane traffic.
  • Loading branch information
@freddygv
freddygv committed Oct 7, 2022
1 parent fac3ddc commit 3034df6
Show file tree
Hide file tree
Showing 11 changed files with 185 additions and 132 deletions.
155 changes: 78 additions & 77 deletions agent/consul/leader_peering_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,6 @@ const (
)

func TestLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T) {
t.Run("without-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeNone)
})
t.Run("manual-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeManual)
})
t.Run("auto-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeAuto)
})
}

func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, mode tlsMode) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
Expand All @@ -64,22 +52,14 @@ func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, mode tlsMode)
c.NodeName = "acceptor"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
if mode == tlsModeManual {
c.ConnectEnabled = false
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
}
if mode == tlsModeAuto {
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}

c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptor.RPC, "dc1")
Expand Down Expand Up @@ -364,18 +344,6 @@ func TestLeader_PeeringSync_Lifecycle_UnexportWhileDown(t *testing.T) {
}

func TestLeader_PeeringSync_Lifecycle_ServerDeletion(t *testing.T) {
t.Run("without-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeNone)
})
t.Run("manual-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeManual)
})
t.Run("auto-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeAuto)
})
}

func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, mode tlsMode) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
Expand All @@ -385,22 +353,14 @@ func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, mode tlsMod
c.NodeName = "acceptor"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
if mode == tlsModeManual {
c.ConnectEnabled = false
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
}
if mode == tlsModeAuto {
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}

c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptor.RPC, "dc1")
Expand Down Expand Up @@ -507,7 +467,7 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) {
t.Run("server-name-validation", func(t *testing.T) {
testLeader_PeeringSync_failsForTLSError(t, func(token *structs.PeeringToken) {
token.ServerName = "wrong.name"
}, `transport: authentication handshake failed: x509: certificate is valid for server.dc1.consul, bob.server.dc1.consul, not wrong.name`)
}, `transport: authentication handshake failed: x509: certificate is valid for server.dc1.peering.11111111-2222-3333-4444-555555555555.consul, not wrong.name`)
})
t.Run("bad-ca-roots", func(t *testing.T) {
wrongRoot, err := ioutil.ReadFile("../../test/client_certs/rootca.crt")
Expand All @@ -522,15 +482,20 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) {
func testLeader_PeeringSync_failsForTLSError(t *testing.T, tokenMutateFn func(token *structs.PeeringToken), expectErr string) {
require.NotNil(t, tokenMutateFn)

ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "bob"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"

c.ConnectEnabled = false
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")

Expand Down Expand Up @@ -573,10 +538,6 @@ func testLeader_PeeringSync_failsForTLSError(t *testing.T, tokenMutateFn func(to
c.NodeName = "betty"
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc2"

c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Betty.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Betty.key"
})
testrpc.WaitForLeader(t, s2.RPC, "dc2")

Expand Down Expand Up @@ -615,11 +576,11 @@ func TestLeader_Peering_DeferredDeletion(t *testing.T) {
t.Skip("too slow for testing.Short")
}

// TODO(peering): Configure with TLS
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "s1.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCTLSPort = freeport.GetOne(t)
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")

Expand Down Expand Up @@ -694,15 +655,21 @@ func TestLeader_Peering_DialerReestablishesConnectionOnError(t *testing.T) {
}

// Reserve a gRPC port so we can restart the accepting server with the same port.
ports := freeport.GetN(t, 1)
acceptingServerPort := ports[0]
acceptingServerPort := freeport.GetOne(t)

ca := connect.TestCA(t, nil)
_, acceptingServer := testServerWithConfig(t, func(c *Config) {
c.NodeName = "acceptingServer.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCPort = acceptingServerPort
c.PeeringEnabled = true
c.GRPCTLSPort = acceptingServerPort
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptingServer.RPC, "dc1")

Expand Down Expand Up @@ -805,9 +772,17 @@ func TestLeader_Peering_DialerReestablishesConnectionOnError(t *testing.T) {
c.NodeName = "acceptingServer.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCPort = acceptingServerPort
c.DataDir = acceptingServer.config.DataDir
c.NodeID = acceptingServer.config.NodeID
c.GRPCTLSPort = acceptingServerPort
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})

testrpc.WaitForLeader(t, acceptingServerRestart.RPC, "dc1")
Expand Down Expand Up @@ -902,11 +877,19 @@ func TestLeader_Peering_ImportedExportedServicesCount(t *testing.T) {
t.Skip("too slow for testing.Short")
}

ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "s1.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.PeeringEnabled = true
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")

Expand Down Expand Up @@ -1204,11 +1187,19 @@ func TestLeader_PeeringMetrics_emitPeeringMetrics(t *testing.T) {
lastIdx = uint64(0)
)

// TODO(peering): Configure with TLS
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "s1.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")

Expand Down Expand Up @@ -1615,10 +1606,20 @@ func Test_Leader_PeeringSync_ServerAddressUpdates(t *testing.T) {
maxRetryBackoff = 1
t.Cleanup(func() { maxRetryBackoff = orig })

ca := connect.TestCA(t, nil)
_, acceptor := testServerWithConfig(t, func(c *Config) {
c.NodeName = "acceptor"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptor.RPC, "dc1")

Expand Down
23 changes: 9 additions & 14 deletions agent/consul/peering_backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,26 +57,21 @@ func (b *PeeringBackend) GetLeaderAddress() string {

// GetTLSMaterials returns the TLS materials for the dialer to dial the acceptor using TLS.
// It returns the server name to validate, and the CA certificate to validate with.
func (b *PeeringBackend) GetTLSMaterials() (string, []string, error) {
// Do not send TLS materials to the dialer if we to not have TLS configured for gRPC.
if b.srv.config.GRPCTLSPort <= 0 && !b.srv.tlsConfigurator.GRPCServerUseTLS() {
return "", nil, nil
}

// If the Connect CA is not in use we rely on the manually configured certs.
// Otherwise we rely on the internally managed server certificate.
if !b.srv.config.ConnectEnabled {
serverName := b.srv.tlsConfigurator.ServerSNI(b.srv.config.Datacenter, "")
caPems := b.srv.tlsConfigurator.GRPCManualCAPems()

return serverName, caPems, nil
func (b *PeeringBackend) GetTLSMaterials(generatingToken bool) (string, []string, error) {
if generatingToken {
if !b.srv.config.ConnectEnabled {
return "", nil, fmt.Errorf("connect.enabled must be set to true in the server's configuration when generating peering tokens")
}
if b.srv.config.GRPCTLSPort <= 0 && !b.srv.tlsConfigurator.GRPCServerUseTLS() {
return "", nil, fmt.Errorf("TLS for gRPC must be enabled when generating peering tokens")
}
}

roots, err := b.srv.getCARoots(nil, b.srv.fsm.State())
if err != nil {
return "", nil, fmt.Errorf("failed to fetch roots: %w", err)
}
if len(roots.Roots) == 0 {
if len(roots.Roots) == 0 || roots.TrustDomain == "" {
return "", nil, fmt.Errorf("CA has not finished initializing")
}

Expand Down
28 changes: 24 additions & 4 deletions agent/consul/peering_backend_oss_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ import (
"github.com/stretchr/testify/require"
gogrpc "google.golang.org/grpc"

"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/proto/pbpeering"
"github.com/hashicorp/consul/sdk/freeport"
"github.com/hashicorp/consul/testrpc"
)

Expand All @@ -21,9 +24,18 @@ func TestPeeringBackend_RejectsPartition(t *testing.T) {
}

t.Parallel()

ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc1"
c.Bootstrap = true
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})

testrpc.WaitForLeader(t, s1.RPC, "dc1")
Expand Down Expand Up @@ -55,9 +67,17 @@ func TestPeeringBackend_IgnoresDefaultPartition(t *testing.T) {
}

t.Parallel()
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc1"
c.Bootstrap = true
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})

testrpc.WaitForLeader(t, s1.RPC, "dc1")
Expand Down
Loading

0 comments on commit 3034df6

Please sign in to comment.