Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[NET-5772] Make tcp external service registered on terminating gw reachable from peered cluster #19881

Merged
merged 5 commits into from
Apr 3, 2024
Merged

[NET-5772] Make tcp external service registered on terminating gw reachable from peered cluster #19881

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7ed7f23
Include SNI + root PEMs from peered cluster on terminating gw filter …
nathancoleman Dec 8, 2023
9c8529b
Abstract existing logic into re-usable function
nathancoleman Dec 11, 2023
5753ee8
Regenerate golden files w/ new listener logic
nathancoleman Feb 8, 2024
17bc757
Add changelog entry
nathancoleman Feb 8, 2024
d0fa205
Use peering bundles that are stable across test runs
nathancoleman Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/19881.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
xds: Make TCP external service registered with terminating gateway reachable from peered cluster
```
70 changes: 46 additions & 24 deletions agent/xds/listeners.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"

"github.com/hashicorp/consul/agent/xds/config"
"github.com/hashicorp/consul/agent/xds/naming"
"github.com/hashicorp/consul/agent/xds/platform"
Expand Down Expand Up @@ -1178,29 +1179,9 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh
}

// Inject peering trust bundles if this service is exported to peered clusters.
if len(peerBundles) > 0 {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This logic gets moved out to a re-usable function below

spiffeConfig, err := makeSpiffeValidatorConfig(
cfgSnap.Roots.TrustDomain,
cfgSnap.RootPEMs(),
peerBundles,
)
if err != nil {
return nil, err
}

typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext)
if !ok {
return nil, fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType)
}

// makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA.
// We nil it out here since the local roots are included in the SPIFFE validator config.
typ.ValidationContext.TrustedCa = nil
typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{
// The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib.
Name: "envoy.tls.cert_validator.spiffe",
TypedConfig: spiffeConfig,
}
err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext, peerBundles)
if err != nil {
return nil, err
}

return makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{
Expand All @@ -1209,6 +1190,32 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh
})
}

func injectSpiffeValidatorConfigForPeers(cfgSnap *proxycfg.ConfigSnapshot, tlsContext *envoy_tls_v3.CommonTlsContext, peerBundles []*pbpeering.PeeringTrustBundle) error {
if len(peerBundles) == 0 {
return nil
}

spiffeConfig, err := makeSpiffeValidatorConfig(cfgSnap.Roots.TrustDomain, cfgSnap.RootPEMs(), peerBundles)
if err != nil {
return err
}

typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext)
if !ok {
return fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType)
}

// makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA.
// We nil it out here since the local roots are included in the SPIFFE validator config.
typ.ValidationContext.TrustedCa = nil
typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{
// The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib.
Name: "envoy.tls.cert_validator.spiffe",
TypedConfig: spiffeConfig,
}
return nil
}

// SPIFFECertValidatorConfig is used to validate certificates from trust domains other than our own.
// With cluster peering we expect peered clusters to have independent certificate authorities.
// This means that we cannot use a single set of root CA certificates to validate client certificates for mTLS,
Expand Down Expand Up @@ -1752,6 +1759,15 @@ type terminatingGatewayFilterChainOpts struct {
}

func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot, tgtwyOpts terminatingGatewayFilterChainOpts) (*envoy_listener_v3.FilterChain, error) {
// We need to at least match the SNI and use the root PEMs from the local cluster; however, requests coming
// from peered clusters where the external service is exported to will have their own SNI and root PEMs.
sniMatches := []string{tgtwyOpts.cluster}
for _, bundle := range tgtwyOpts.peerTrustBundles {
svc := tgtwyOpts.service
sourceSNI := connect.PeeredServiceSNI(svc.Name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), bundle.PeerName, cfgSnap.Roots.TrustDomain)
sniMatches = append(sniMatches, sourceSNI)
}

tlsContext := &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(
cfgSnap.TerminatingGateway.ServiceLeaves[tgtwyOpts.service],
Expand All @@ -1760,13 +1776,19 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.
),
RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
}

err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext.CommonTlsContext, tgtwyOpts.peerTrustBundles)
if err != nil {
return nil, err
}

transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
if err != nil {
return nil, err
}

filterChain := &envoy_listener_v3.FilterChain{
FilterChainMatch: makeSNIFilterChainMatch(tgtwyOpts.cluster),
FilterChainMatch: makeSNIFilterChainMatch(sniMatches...),
Filters: make([]*envoy_listener_v3.Filter, 0, 3),
TransportSocket: transportSocket,
}
Expand Down
28 changes: 10 additions & 18 deletions agent/xds/resources_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"github.com/hashicorp/consul/agent/xds/testcommon"
"github.com/hashicorp/consul/agent/xdsv2"
"github.com/hashicorp/consul/envoyextensions/xdscommon"
"github.com/hashicorp/consul/proto/private/pbpeering"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/consul/types"
)
Expand Down Expand Up @@ -2281,32 +2280,25 @@ func getTerminatingGatewayPeeringGoldenTestCases() []goldenTestCase {
{
name: "terminating-gateway-with-peer-trust-bundle",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
roots, _ := proxycfg.TestCerts(t)
bundles := proxycfg.TestPeerTrustBundles(t)
return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, []proxycfg.UpdateEvent{
{
CorrelationID: "peer-trust-bundle:web",
Result: &pbpeering.TrustBundleListByServiceResponse{
Bundles: []*pbpeering.PeeringTrustBundle{
{
TrustDomain: "foo.bar.gov",
PeerName: "dc2",
Partition: "default",
RootPEMs: []string{
roots.Roots[0].RootCert,
},
ExportedPartition: "default",
CreateIndex: 0,
ModifyIndex: 0,
},
},
},
Result: bundles,
},
{
CorrelationID: "service-intentions:web",
Result: structs.SimplifiedIntentions{
{
SourceName: "source",
SourcePeer: "dc2",
SourcePeer: bundles.Bundles[0].PeerName,
DestinationName: "web",
DestinationPartition: "default",
Action: structs.IntentionActionAllow,
},
{
SourceName: "source",
SourcePeer: bundles.Bundles[1].PeerName,
DestinationName: "web",
DestinationPartition: "default",
Action: structs.IntentionActionAllow,
Expand Down
42 changes: 38 additions & 4 deletions agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,9 @@
{
"filterChainMatch": {
"serverNames": [
"web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
"web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"web.default.default.peer-a.external.11111111-2222-3333-4444-555555555555.consul",
"web.default.default.peer-b.external.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
Expand All @@ -184,7 +186,16 @@
"authenticated": {
"principalName": {
"safeRegex": {
"regex": "^spiffe://foo.bar.gov/ns/default/dc/[^/]+/svc/source$"
"regex": "^spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/[^/]+/svc/source$"
}
}
}
},
{
"authenticated": {
"principalName": {
"safeRegex": {
"regex": "^spiffe://d89ac423-e95a-475d-94f2-1c557c57bf31.consul/ns/default/dc/[^/]+/svc/source$"
}
}
}
Expand Down Expand Up @@ -222,8 +233,31 @@
],
"tlsParams": {},
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
"customValidatorConfig": {
"name": "envoy.tls.cert_validator.spiffe",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig",
"trustDomains": [
{
"name": "11111111-2222-3333-4444-555555555555.consul",
"trustBundle": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
},
{
"name": "1c053652-8512-4373-90cf-5a7f6263a994.consul",
"trustBundle": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n"
}
},
{
"name": "d89ac423-e95a-475d-94f2-1c557c57bf31.consul",
"trustBundle": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICcTCCAdoCCQDyGxC08cD0BDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFkMQwwCgYDVQQKDANGb28x\nEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXItYjEdMBsGCSqGSIb3DQEJ\nARYOZm9vQHBlZXItYi5jb20wHhcNMjIwNTI2MDExNjE2WhcNMjMwNTI2MDExNjE2\nWjB9MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFk\nMQwwCgYDVQQKDANGb28xEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXIt\nYjEdMBsGCSqGSIb3DQEJARYOZm9vQHBlZXItYi5jb20wgZ8wDQYJKoZIhvcNAQEB\nBQADgY0AMIGJAoGBAL4i5erdZ5vKk3mzW9Qt6Wvw/WN/IpMDlL0a28wz9oDCtMLN\ncD/XQB9yT5jUwb2s4mD1lCDZtee8MHeD8zygICozufWVB+u2KvMaoA50T9GMQD0E\nz/0nz/Z703I4q13VHeTpltmEpYcfxw/7nJ3leKA34+Nj3zteJ70iqvD/TNBBAgMB\nAAEwDQYJKoZIhvcNAQELBQADgYEAbL04gicH+EIznDNhZJEb1guMBtBBJ8kujPyU\nao8xhlUuorDTLwhLpkKsOhD8619oSS8KynjEBichidQRkwxIaze0a2mrGT+tGBMf\npVz6UeCkqpde6bSJ/ozEe/2seQzKqYvRT1oUjLwYvY7OIh2DzYibOAxh6fewYAmU\n5j5qNLc=\n-----END CERTIFICATE-----\n"
}
}
]
}
}
}
},
Expand Down