Skip to content

Commit

Permalink
support loading route configuration via rds (#4098)
Browse files Browse the repository at this point in the history 
* support loading route configuration via rds

* fix any shadowing

* fix test

* add fully static option

* support dynamically defined rds

* fix build

* downgrade opa
  • Loading branch information
@calebdoxsey
calebdoxsey committed Apr 17, 2023
1 parent d485ca8 commit f63945c
Show file tree
Hide file tree
Showing 10 changed files with 300 additions and 672 deletions.
105 changes: 98 additions & 7 deletions config/envoyconfig/bootstrap.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package envoyconfig

import (
"context"
"fmt"
"os"
"path/filepath"
Expand All @@ -25,6 +26,47 @@ var (
envoyAdminClusterName = "pomerium-envoy-admin"
)

// BuildBootstrap builds the bootstrap config.
func (b *Builder) BuildBootstrap(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (bootstrap *envoy_config_bootstrap_v3.Bootstrap, err error) {
bootstrap = new(envoy_config_bootstrap_v3.Bootstrap)

bootstrap.Admin, err = b.BuildBootstrapAdmin(cfg)
if err != nil {
return nil, fmt.Errorf("error building bootstrap admin: %w", err)
}

bootstrap.DynamicResources, err = b.BuildBootstrapDynamicResources(cfg, fullyStatic)
if err != nil {
return nil, fmt.Errorf("error building bootstrap dynamic resources: %w", err)
}

bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime()
if err != nil {
return nil, fmt.Errorf("error building bootstrap layered runtime: %w", err)
}

bootstrap.Node = &envoy_config_core_v3.Node{
Id: telemetry.ServiceName(cfg.Options.Services),
Cluster: telemetry.ServiceName(cfg.Options.Services),
}

bootstrap.StaticResources, err = b.BuildBootstrapStaticResources(ctx, cfg, fullyStatic)
if err != nil {
return nil, fmt.Errorf("error building bootstrap static resources: %w", err)
}

bootstrap.StatsConfig, err = b.BuildBootstrapStatsConfig(cfg)
if err != nil {
return nil, err
}

return bootstrap, nil
}

// BuildBootstrapAdmin builds the admin config for the envoy bootstrap.
func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (admin *envoy_config_bootstrap_v3.Admin, err error) {
admin = &envoy_config_bootstrap_v3.Admin{
Expand Down Expand Up @@ -53,6 +95,39 @@ func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (admin *envoy_config_b
return admin, nil
}

// BuildBootstrapDynamicResources builds the dynamic resources for the envoy bootstrap.
func (b *Builder) BuildBootstrapDynamicResources(
cfg *config.Config,
fullyStatic bool,
) (dynamicResources *envoy_config_bootstrap_v3.Bootstrap_DynamicResources, err error) {
if fullyStatic {
return nil, nil
}
return &envoy_config_bootstrap_v3.Bootstrap_DynamicResources{
AdsConfig: &envoy_config_core_v3.ApiConfigSource{
ApiType: envoy_config_core_v3.ApiConfigSource_ApiType(envoy_config_core_v3.ApiConfigSource_ApiType_value["DELTA_GRPC"]),
TransportApiVersion: envoy_config_core_v3.ApiVersion_V3,
GrpcServices: []*envoy_config_core_v3.GrpcService{
{
TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
ClusterName: "pomerium-control-plane-grpc",
},
},
},
},
},
LdsConfig: &envoy_config_core_v3.ConfigSource{
ResourceApiVersion: envoy_config_core_v3.ApiVersion_V3,
ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{},
},
CdsConfig: &envoy_config_core_v3.ConfigSource{
ResourceApiVersion: envoy_config_core_v3.ApiVersion_V3,
ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{},
},
}, nil
}

// BuildBootstrapLayeredRuntime builds the layered runtime for the envoy bootstrap.
func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.LayeredRuntime, error) {
layer, err := structpb.NewStruct(map[string]interface{}{
Expand All @@ -78,7 +153,27 @@ func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.Lay

// BuildBootstrapStaticResources builds the static resources for the envoy bootstrap. It includes the control plane
// cluster.
func (b *Builder) BuildBootstrapStaticResources() (*envoy_config_bootstrap_v3.Bootstrap_StaticResources, error) {
func (b *Builder) BuildBootstrapStaticResources(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (staticResources *envoy_config_bootstrap_v3.Bootstrap_StaticResources, err error) {
staticResources = new(envoy_config_bootstrap_v3.Bootstrap_StaticResources)

if fullyStatic {
staticResources.Clusters, err = b.BuildClusters(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("error building clusters: %w", err)
}

staticResources.Listeners, err = b.BuildListeners(ctx, cfg, fullyStatic)
if err != nil {
return nil, fmt.Errorf("error building listeners: %w", err)
}

return staticResources, nil
}

grpcAddr, err := parseAddress(b.localGRPCAddress)
if err != nil {
return nil, fmt.Errorf("envoyconfig: invalid local gRPC address: %w", err)
Expand Down Expand Up @@ -114,13 +209,9 @@ func (b *Builder) BuildBootstrapStaticResources() (*envoy_config_bootstrap_v3.Bo
TypedExtensionProtocolOptions: buildTypedExtensionProtocolOptions(nil, upstreamProtocolHTTP2),
}

staticCfg := &envoy_config_bootstrap_v3.Bootstrap_StaticResources{
Clusters: []*envoy_config_cluster_v3.Cluster{
controlPlaneCluster,
},
}
staticResources.Clusters = append(staticResources.Clusters, controlPlaneCluster)

return staticCfg, nil
return staticResources, nil
}

// BuildBootstrapStatsConfig builds a the stats config the envoy bootstrap.
Expand Down
5 changes: 3 additions & 2 deletions config/envoyconfig/bootstrap_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package envoyconfig

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -51,7 +52,7 @@ func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
t.Run("valid", func(t *testing.T) {
b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
staticCfg, err := b.BuildBootstrapStaticResources()
staticCfg, err := b.BuildBootstrapStaticResources(context.Background(), &config.Config{}, false)
assert.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `
{
Expand Down Expand Up @@ -95,7 +96,7 @@ func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
})
t.Run("bad gRPC address", func(t *testing.T) {
b := New("xyz:zyx", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
_, err := b.BuildBootstrapStaticResources()
_, err := b.BuildBootstrapStaticResources(context.Background(), &config.Config{}, false)
assert.Error(t, err)
})
}
Expand Down
145 changes: 57 additions & 88 deletions config/envoyconfig/listeners.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,15 @@ func init() {
}

// BuildListeners builds envoy listeners from the given config.
func (b *Builder) BuildListeners(ctx context.Context, cfg *config.Config) ([]*envoy_config_listener_v3.Listener, error) {
func (b *Builder) BuildListeners(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) ([]*envoy_config_listener_v3.Listener, error) {
var listeners []*envoy_config_listener_v3.Listener

if config.IsAuthenticate(cfg.Options.Services) || config.IsProxy(cfg.Options.Services) {
li, err := b.buildMainListener(ctx, cfg)
li, err := b.buildMainListener(ctx, cfg, fullyStatic)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -128,7 +132,11 @@ func (b *Builder) buildTLSSocket(ctx context.Context, cfg *config.Config, certs
}, nil
}

func (b *Builder) buildMainListener(ctx context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
func (b *Builder) buildMainListener(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Listener, error) {
li := newEnvoyListener("http-ingress")
if cfg.Options.UseProxyProtocol {
li.ListenerFilters = append(li.ListenerFilters, ProxyProtocolFilter())
Expand All @@ -137,7 +145,7 @@ func (b *Builder) buildMainListener(ctx context.Context, cfg *config.Config) (*e
if cfg.Options.InsecureServer {
li.Address = buildAddress(cfg.Options.Addr, 80)

filter, err := b.buildMainHTTPConnectionManagerFilter(cfg.Options)
filter, err := b.buildMainHTTPConnectionManagerFilter(ctx, cfg, fullyStatic)
if err != nil {
return nil, err
}
Expand All @@ -156,7 +164,7 @@ func (b *Builder) buildMainListener(ctx context.Context, cfg *config.Config) (*e
return nil, err
}

filter, err := b.buildMainHTTPConnectionManagerFilter(cfg.Options, allCertificates...)
filter, err := b.buildMainHTTPConnectionManagerFilter(ctx, cfg, fullyStatic)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -254,67 +262,13 @@ func (b *Builder) buildMetricsListener(cfg *config.Config) (*envoy_config_listen
}

func (b *Builder) buildMainHTTPConnectionManagerFilter(
options *config.Options,
certs ...tls.Certificate,
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Filter, error) {
authorizeURLs, err := options.GetInternalAuthorizeURLs()
if err != nil {
return nil, err
}

dataBrokerURLs, err := options.GetInternalDataBrokerURLs()
if err != nil {
return nil, err
}

allHosts, err := getAllRouteableHosts(options, options.Addr)
if err != nil {
return nil, err
}

var virtualHosts []*envoy_config_route_v3.VirtualHost
for _, host := range allHosts {
requireStrictTransportSecurity := cryptutil.HasCertificateForServerName(certs, host)
vh, err := b.buildVirtualHost(options, host, host, requireStrictTransportSecurity)
if err != nil {
return nil, err
}

if options.Addr == options.GetGRPCAddr() {
// if this is a gRPC service domain and we're supposed to handle that, add those routes
if (config.IsAuthorize(options.Services) && urlsMatchHost(authorizeURLs, host)) ||
(config.IsDataBroker(options.Services) && urlsMatchHost(dataBrokerURLs, host)) {
rs, err := b.buildGRPCRoutes()
if err != nil {
return nil, err
}
vh.Routes = append(vh.Routes, rs...)
}
}

// if we're the proxy, add all the policy routes
if config.IsProxy(options.Services) {
rs, err := b.buildPolicyRoutes(options, host)
if err != nil {
return nil, err
}
vh.Routes = append(vh.Routes, rs...)
}

if len(vh.Routes) > 0 {
virtualHosts = append(virtualHosts, vh)
}
}

vh, err := b.buildVirtualHost(options, "catch-all", "*", false)
if err != nil {
return nil, err
}
virtualHosts = append(virtualHosts, vh)

var grpcClientTimeout *durationpb.Duration
if options.GRPCClientTimeout != 0 {
grpcClientTimeout = durationpb.New(options.GRPCClientTimeout)
if cfg.Options.GRPCClientTimeout != 0 {
grpcClientTimeout = durationpb.New(cfg.Options.GRPCClientTimeout)
} else {
grpcClientTimeout = durationpb.New(30 * time.Second)
}
Expand All @@ -329,45 +283,59 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
filters = append(filters, HTTPRouterFilter())

var maxStreamDuration *durationpb.Duration
if options.WriteTimeout > 0 {
maxStreamDuration = durationpb.New(options.WriteTimeout)
if cfg.Options.WriteTimeout > 0 {
maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout)
}

rc, err := b.buildRouteConfiguration("main", virtualHosts)
if err != nil {
return nil, err
}
tracingProvider, err := buildTracingHTTP(options)
tracingProvider, err := buildTracingHTTP(cfg.Options)
if err != nil {
return nil, err
}

return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
mgr := &envoy_http_connection_manager.HttpConnectionManager{
AlwaysSetRequestIdInResponse: true,

CodecType: options.GetCodecType().ToEnvoy(),
StatPrefix: "ingress",
RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: rc,
},
HttpFilters: filters,
AccessLog: buildAccessLogs(options),
CodecType: cfg.Options.GetCodecType().ToEnvoy(),
StatPrefix: "ingress",
HttpFilters: filters,
AccessLog: buildAccessLogs(cfg.Options),
CommonHttpProtocolOptions: &envoy_config_core_v3.HttpProtocolOptions{
IdleTimeout: durationpb.New(options.IdleTimeout),
IdleTimeout: durationpb.New(cfg.Options.IdleTimeout),
MaxStreamDuration: maxStreamDuration,
},
HttpProtocolOptions: http1ProtocolOptions,
RequestTimeout: durationpb.New(options.ReadTimeout),
RequestTimeout: durationpb.New(cfg.Options.ReadTimeout),
Tracing: &envoy_http_connection_manager.HttpConnectionManager_Tracing{
RandomSampling: &envoy_type_v3.Percent{Value: options.TracingSampleRate * 100},
RandomSampling: &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100},
Provider: tracingProvider,
},
// See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
UseRemoteAddress: &wrappers.BoolValue{Value: true},
SkipXffAppend: options.SkipXffAppend,
XffNumTrustedHops: options.XffNumTrustedHops,
LocalReplyConfig: b.buildLocalReplyConfig(options, false),
}), nil
SkipXffAppend: cfg.Options.SkipXffAppend,
XffNumTrustedHops: cfg.Options.XffNumTrustedHops,
LocalReplyConfig: b.buildLocalReplyConfig(cfg.Options, false),
}

if fullyStatic {
routeConfiguration, err := b.buildMainRouteConfiguration(ctx, cfg)
if err != nil {
return nil, err
}
mgr.RouteSpecifier = &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: routeConfiguration,
}
} else {
mgr.RouteSpecifier = &envoy_http_connection_manager.HttpConnectionManager_Rds{
Rds: &envoy_http_connection_manager.Rds{
ConfigSource: &envoy_config_core_v3.ConfigSource{
ResourceApiVersion: envoy_config_core_v3.ApiVersion_V3,
ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{},
},
RouteConfigName: "main",
},
}
}

return HTTPConnectionManagerFilter(mgr), nil
}

func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) {
Expand Down Expand Up @@ -546,7 +514,8 @@ func (b *Builder) buildDownstreamTLSContextMulti(
TlsCertificates: envoyCerts,
AlpnProtocols: getALPNProtos(cfg.Options),
ValidationContextType: b.buildDownstreamValidationContext(ctx, cfg),
}}, nil
},
}, nil
}

func getALPNProtos(opts *config.Options) []string {
Expand Down
Loading

0 comments on commit f63945c

Please sign in to comment.