expose envoy cluster options in policy (#1804)

config/constants.go

@@ -1,13 +1,21 @@
 package config
 
-import "errors"
+import (
+	"errors"
+
+	"google.golang.org/protobuf/encoding/protojson"
+)
 
 const (
-	policyKey      = "policy"
-	toKey          = "to"
-	healthCheckKey = "health_check"
+	policyKey    = "policy"
+	toKey        = "to"
+	envoyOptsKey = "_envoy_opts"
 )
 
 var (
 	errKeysMustBeStrings = errors.New("cannot convert nested map: all keys must be strings")
 )
+
+var (
+	protoPartial = protojson.UnmarshalOptions{AllowPartial: true, DiscardUnknown: true}
+)

config/custom.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"reflect"
 
-	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	"github.com/mitchellh/mapstructure"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
@@ -117,14 +117,13 @@ func DecodeOptionsHookFunc() mapstructure.DecodeHookFunc {
 			if !ok {
 				continue
 			}
-			raw, ok := pm[healthCheckKey]
-			if ok {
-				hc := new(envoy_config_core_v3.HealthCheck)
-				if err := parseJSONPB(raw, hc); err != nil {
-					return nil, fmt.Errorf("%s: %w", healthCheckKey, err)
-				}
-				pm[healthCheckKey] = hc
+
+			envoyOpts, err := parseEnvoyClusterOpts(pm)
+			if err != nil {
+				return nil, err
 			}
+			pm[envoyOptsKey] = envoyOpts
+
 			rawTo, ok := pm[toKey]
 			if !ok {
 				continue
@@ -145,9 +144,20 @@ func DecodeOptionsHookFunc() mapstructure.DecodeHookFunc {
 	}
 }
 
+// parseEnvoyClusterOpts parses src as envoy cluster spec https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
+// on top of some pre-filled default values
+func parseEnvoyClusterOpts(src interface{}) (*envoy_config_cluster_v3.Cluster, error) {
+	c := new(envoy_config_cluster_v3.Cluster)
+	if err := parseJSONPB(src, c, protoPartial); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
 // parseJSONPB takes an intermediate representation and parses it using protobuf parser
 // that correctly handles oneof and other data types
-func parseJSONPB(raw interface{}, dst proto.Message) error {
+func parseJSONPB(raw interface{}, dst proto.Message, opts protojson.UnmarshalOptions) error {
 	ms, err := serializable(raw)
 	if err != nil {
 		return err
@@ -158,7 +168,7 @@ func parseJSONPB(raw interface{}, dst proto.Message) error {
 		return err
 	}
 
-	return protojson.Unmarshal(data, dst)
+	return opts.Unmarshal(data, dst)
 }
 
 // serializable converts mapstructure nested map into map[string]interface{} that is serializable to JSON

config/options.go

@@ -332,7 +332,7 @@ func NewDefaultOptions() *Options {
 func newOptionsFromConfig(configFile string) (*Options, error) {
 	o, err := optionsFromViper(configFile)
 	if err != nil {
-		return nil, fmt.Errorf("config: options from config file %w", err)
+		return nil, fmt.Errorf("config: options from config file %q: %w", configFile, err)
 	}
 	serviceName := telemetry.ServiceName(o.Services)
 	metrics.AddPolicyCountCallback(serviceName, func() int64 {

config/policy.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
-	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/ptypes"
 
 	"github.com/pomerium/pomerium/internal/hashutil"
@@ -139,13 +138,9 @@ type Policy struct {
 	// to upstream requests.
 	EnableGoogleCloudServerlessAuthentication bool `mapstructure:"enable_google_cloud_serverless_authentication" yaml:"enable_google_cloud_serverless_authentication,omitempty"` //nolint
 
-	// OutlierDetection configures outlier detection for the upstream cluster.
-	OutlierDetection *PolicyOutlierDetection `mapstructure:"outlier_detection" yaml:"outlier_detection,omitempty" json:"outlier_detection,omitempty"`
-
-	// HealthCheck defines active health checks. See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/health_check.proto
-	HealthCheck *envoy_config_core_v3.HealthCheck `mapstructure:"health_check" yaml:"health_check,omitempty" json:"health_check,omitempty"`
-
 	SubPolicies []SubPolicy `mapstructure:"sub_policies" yaml:"sub_policies,omitempty" json:"sub_policies,omitempty"`
+
+	EnvoyOpts *envoy_config_cluster_v3.Cluster `mapstructure:"_envoy_opts" yaml:"-" json:"-"`
 }
 
 // A SubPolicy is a protobuf Policy within a protobuf Route.
@@ -221,30 +216,7 @@ func NewPolicyFromProto(pb *configpb.Route) (*Policy, error) {
 			StripQuery:     pb.Redirect.StripQuery,
 		}
 	}
-	if pb.OutlierDetection != nil {
-		p.OutlierDetection = &PolicyOutlierDetection{
-			Consecutive_5Xx:                        pb.OutlierDetection.Consecutive_5Xx,
-			Interval:                               pb.OutlierDetection.Interval,
-			BaseEjectionTime:                       pb.OutlierDetection.BaseEjectionTime,
-			MaxEjectionPercent:                     pb.OutlierDetection.MaxEjectionPercent,
-			EnforcingConsecutive_5Xx:               pb.OutlierDetection.EnforcingConsecutive_5Xx,
-			EnforcingSuccessRate:                   pb.OutlierDetection.EnforcingSuccessRate,
-			SuccessRateMinimumHosts:                pb.OutlierDetection.SuccessRateMinimumHosts,
-			SuccessRateRequestVolume:               pb.OutlierDetection.SuccessRateRequestVolume,
-			SuccessRateStdevFactor:                 pb.OutlierDetection.SuccessRateStdevFactor,
-			ConsecutiveGatewayFailure:              pb.OutlierDetection.ConsecutiveGatewayFailure,
-			EnforcingConsecutiveGatewayFailure:     pb.OutlierDetection.EnforcingConsecutiveGatewayFailure,
-			SplitExternalLocalOriginErrors:         pb.OutlierDetection.SplitExternalLocalOriginErrors,
-			ConsecutiveLocalOriginFailure:          pb.OutlierDetection.ConsecutiveLocalOriginFailure,
-			EnforcingConsecutiveLocalOriginFailure: pb.OutlierDetection.EnforcingConsecutiveLocalOriginFailure,
-			EnforcingLocalOriginSuccessRate:        pb.OutlierDetection.EnforcingLocalOriginSuccessRate,
-			FailurePercentageThreshold:             pb.OutlierDetection.FailurePercentageThreshold,
-			EnforcingFailurePercentage:             pb.OutlierDetection.EnforcingFailurePercentage,
-			EnforcingFailurePercentageLocalOrigin:  pb.OutlierDetection.EnforcingFailurePercentageLocalOrigin,
-			FailurePercentageMinimumHosts:          pb.OutlierDetection.FailurePercentageMinimumHosts,
-			FailurePercentageRequestVolume:         pb.OutlierDetection.FailurePercentageRequestVolume,
-		}
-	}
+
 	for _, sp := range pb.GetPolicies() {
 		p.SubPolicies = append(p.SubPolicies, SubPolicy{
 			ID:               sp.GetId(),
@@ -320,30 +292,7 @@ func (p *Policy) ToProto() *configpb.Route {
 			StripQuery:     p.Redirect.StripQuery,
 		}
 	}
-	if p.OutlierDetection != nil {
-		pb.OutlierDetection = &configpb.OutlierDetection{
-			Consecutive_5Xx:                        p.OutlierDetection.Consecutive_5Xx,
-			Interval:                               p.OutlierDetection.Interval,
-			BaseEjectionTime:                       p.OutlierDetection.BaseEjectionTime,
-			MaxEjectionPercent:                     p.OutlierDetection.MaxEjectionPercent,
-			EnforcingConsecutive_5Xx:               p.OutlierDetection.EnforcingConsecutive_5Xx,
-			EnforcingSuccessRate:                   p.OutlierDetection.EnforcingSuccessRate,
-			SuccessRateMinimumHosts:                p.OutlierDetection.SuccessRateMinimumHosts,
-			SuccessRateRequestVolume:               p.OutlierDetection.SuccessRateRequestVolume,
-			SuccessRateStdevFactor:                 p.OutlierDetection.SuccessRateStdevFactor,
-			ConsecutiveGatewayFailure:              p.OutlierDetection.ConsecutiveGatewayFailure,
-			EnforcingConsecutiveGatewayFailure:     p.OutlierDetection.EnforcingConsecutiveGatewayFailure,
-			SplitExternalLocalOriginErrors:         p.OutlierDetection.SplitExternalLocalOriginErrors,
-			ConsecutiveLocalOriginFailure:          p.OutlierDetection.ConsecutiveLocalOriginFailure,
-			EnforcingConsecutiveLocalOriginFailure: p.OutlierDetection.EnforcingConsecutiveLocalOriginFailure,
-			EnforcingLocalOriginSuccessRate:        p.OutlierDetection.EnforcingLocalOriginSuccessRate,
-			FailurePercentageThreshold:             p.OutlierDetection.FailurePercentageThreshold,
-			EnforcingFailurePercentage:             p.OutlierDetection.EnforcingFailurePercentage,
-			EnforcingFailurePercentageLocalOrigin:  p.OutlierDetection.EnforcingFailurePercentageLocalOrigin,
-			FailurePercentageMinimumHosts:          p.OutlierDetection.FailurePercentageMinimumHosts,
-			FailurePercentageRequestVolume:         p.OutlierDetection.FailurePercentageRequestVolume,
-		}
-	}
+
 	return pb
 }
 
@@ -433,12 +382,6 @@ func (p *Policy) Validate() error {
 		return fmt.Errorf("config: only prefix_rewrite or regex_rewrite_pattern can be specified, but not both")
 	}
 
-	if p.HealthCheck != nil {
-		if err := p.HealthCheck.Validate(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 

docs/docs/topics/load_balancing.md

@@ -0,0 +1,60 @@
+---
+title: Upstream Load Balancing
+description: >-
+  This article covers Pomerium built-in load balancing capabilities in presence of multiple upstreams.
+---
+
+# Upstream Load Balancing
+
+This article covers Pomerium built-in load balancing capabilities in presence of multiple upstreams.
+
+## Multiple Upstreams
+
+You may specify multiple servers for your upstream application, and Pomerium would load balance user requests between them.
+
+```yaml
+policy:
+  - from: https://myapp.localhost.pomerium.io
+    to:
+      - http://myapp-srv-1:8080
+      - http://myapp-srv-2:8080
+```
+
+::: tip
+    In presence of multiple upstreams, make sure to specify either an active or passive health check, or both, to avoid requests served to unhealthy backend.
+:::
+
+### Active Health Checks
+
+Active health checks issue periodic requests to each upstream to determine its health.
+See [Health Checking](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking) for a comprehensive overview.
+
+```yaml
+policy:
+  - from: https://myapp.localhost.pomerium.io
+    to:
+      - http://myapp-srv-1:8080
+      - http://myapp-srv-2:8080
+    health_checks:
+      - timeout: 10s
+        interval: 60s
+        healthy_threshold: 1
+        unhealthy_threshold: 2
+        http_health_check:
+          path: "/"
+```
+### Passive Health Checks
+
+Passive health check tries to deduce upstream server health based on recent observed responses.
+See [Outlier Detection](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/outlier) for a comprehensive overview.
+
+## Load Balancing Method
+
+`lb_policy` should be set to one of the values:
+
+- [`ROUND_ROBIN`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-round-robin) (default)
+- [`LEAST_REQUEST`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-least-request) and may be further configured using [``](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-leastrequestlbconfig)
+- [`RING_HASH`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash) and may be further configured using [`ring_hash_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-ringhashlbconfig) option
+- [`RANDOM`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#random)
+- [`MAGLEV`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#maglev) and may be further configured using [`maglev_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-maglevlbconfig) option
+

docs/reference/readme.md

@@ -1370,12 +1370,28 @@ When enabled, this option will pass identity headers to upstream applications. T
 If set, enables proxying of SPDY protocol upgrades.
 
 
-### Health Check
-- Config File Key: `health_check`
-- Type: `object`
+### Load Balancing
+- Config File Key: `lb_policy`
+- Type: `enum`
+- Optional
+
+In presence of multiple upstreams, defines load balancing strategy between them.
+
+- [`ROUND_ROBIN`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-round-robin) (default)
+- [`LEAST_REQUEST`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-least-request) and may be further configured using [``](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-leastrequestlbconfig)
+- [`RING_HASH`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash) and may be further configured using [`ring_hash_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-ringhashlbconfig) option
+- [`RANDOM`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#random)
+- [`MAGLEV`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#maglev) and may be further configured using [`maglev_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-maglevlbconfig) option
+
+
+### Health Checks
+- Config File Key: `health_checks`
+- Type: `array of objects`
 - Optional
 
-When defined, will issue periodic health check requests to upstream servers.
+When defined, will issue periodic health check requests to upstream servers. When health checks are defined, unhealthy upstream servers would not serve traffic.
+See also `outlier_detection` for automatic upstream server health detection.
+In presence of multiple upstream servers, it is recommended to set up either `health_checks` or `outlier_detection` or both.
 See [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking) for a list of supported parameters.
 
 

docs/reference/settings.yaml

@@ -1502,14 +1502,30 @@ settings:
           - Default: `false`
         doc: |
           If set, enables proxying of SPDY protocol upgrades.
-      - name: "Health Check"
-        keys: ["health_check"]
+      - name: "Load Balancing"
+        keys: ["lb_policy"]
         attributes: |
-          - Config File Key: `health_check`
-          - Type: `object`
+          - Config File Key: `lb_policy`
+          - Type: `enum`
+          - Optional
+        doc: |
+          In presence of multiple upstreams, defines load balancing strategy between them.
+
+          - [`ROUND_ROBIN`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-round-robin) (default)
+          - [`LEAST_REQUEST`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-least-request) and may be further configured using [``](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-leastrequestlbconfig)
+          - [`RING_HASH`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash) and may be further configured using [`ring_hash_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-ringhashlbconfig) option
+          - [`RANDOM`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#random)
+          - [`MAGLEV`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#maglev) and may be further configured using [`maglev_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-maglevlbconfig) option
+      - name: "Health Checks"
+        keys: ["health_checks"]
+        attributes: |
+          - Config File Key: `health_checks`
+          - Type: `array of objects`
           - Optional
         doc: |
-          When defined, will issue periodic health check requests to upstream servers.
+          When defined, will issue periodic health check requests to upstream servers. When health checks are defined, unhealthy upstream servers would not serve traffic.
+          See also `outlier_detection` for automatic upstream server health detection.
+          In presence of multiple upstream servers, it is recommended to set up either `health_checks` or `outlier_detection` or both.
           See [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking) for a list of supported parameters.
       - name: "Websocket Connections"
         keys: ["allow_websockets"]

internal/cmd/pomerium/pomerium.go

@@ -64,8 +64,15 @@ func Run(ctx context.Context, configFile string) error {
 	if err != nil {
 		return fmt.Errorf("error creating control plane: %w", err)
 	}
-	src.OnConfigChange(controlPlane.OnConfigChange)
-	controlPlane.OnConfigChange(src.GetConfig())
+	src.OnConfigChange(func(cfg *config.Config) {
+		if err := controlPlane.OnConfigChange(cfg); err != nil {
+			log.Error().Err(err).Msg("config change")
+		}
+	})
+
+	if err = controlPlane.OnConfigChange(src.GetConfig()); err != nil {
+		return fmt.Errorf("applying config: %w", err)
+	}
 
 	_, grpcPort, _ := net.SplitHostPort(controlPlane.GRPCListener.Addr().String())
 	_, httpPort, _ := net.SplitHostPort(controlPlane.HTTPListener.Addr().String())

internal/controlplane/constants.go

@@ -0,0 +1,23 @@
+package controlplane
+
+import (
+	"errors"
+	"time"
+
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	"github.com/golang/protobuf/ptypes"
+)
+
+var (
+	errNoEndpoints           = errors.New("cluster must have endpoints")
+	defaultConnectionTimeout = ptypes.DurationProto(time.Second * 10)
+)
+
+// newDefaultEnvoyClusterConfig creates envoy cluster with certain default values
+func newDefaultEnvoyClusterConfig() *envoy_config_cluster_v3.Cluster {
+	return &envoy_config_cluster_v3.Cluster{
+		ConnectTimeout:  defaultConnectionTimeout,
+		RespectDnsTtl:   true,
+		DnsLookupFamily: envoy_config_cluster_v3.Cluster_AUTO,
+	}
+}