[backplane] add overload manager

cmd/backplane/main.go

@@ -87,6 +87,9 @@ var (
 
 	useEnvoyContrib = flag.Bool("use_envoy_contrib", false, "Use Envoy contrib filters.")
 
+	overloadMaxHeapSizeBytes       = flag.Uint64("overload-max-heap-size-bytes", 0, "Maximum heap size in bytes for Envoy overload manager.")
+	overloadMaxActiveConnections = flag.Uint64("overload-max-active-connections", 0, "Maximum number of active downstream connections for Envoy overload manager.")
+
 	k8sKVNamespace    = flag.String("k8s_kv_namespace", os.Getenv("POD_NAMESPACE"), "Namespace for the K/V store.")
 	k8sKVPeerSelector = flag.String("k8s_kv_peer_selector", "app.kubernetes.io/component=backplane", "Label selector for K/V store peers.")
 
@@ -283,6 +286,12 @@ func main() {
 	if *useEnvoyContrib {
 		proxyOpts = append(proxyOpts, bpctrl.WithEnvoyContrib())
 	}
+	if *overloadMaxHeapSizeBytes > 0 {
+		proxyOpts = append(proxyOpts, bpctrl.WithOverloadMaxHeapSizeBytes(*overloadMaxHeapSizeBytes))
+	}
+	if *overloadMaxActiveConnections > 0 {
+		proxyOpts = append(proxyOpts, bpctrl.WithOverloadMaxActiveConnections(*overloadMaxActiveConnections))
+	}
 	if *readyProbePort != 0 {
 		hc := healthchecker.NewAggregatedHealthChecker()
 		go hc.Start(ctx, *readyProbePort)

pkg/backplane/controllers/proxy.go

@@ -59,13 +59,15 @@ type ProxyReconciler struct {
 }
 
 type options struct {
-	chConn                   clickhouse.Conn
-	chOpts                   *clickhouse.Options
-	apiServerTLSClientConfig *tls.Config
-	goPluginDir              string
-	releaseURL               string
-	useEnvoyContrib          bool
-	healthChecker            *healthchecker.AggregatedHealthChecker
+	chConn                          clickhouse.Conn
+	chOpts                          *clickhouse.Options
+	apiServerTLSClientConfig        *tls.Config
+	goPluginDir                     string
+	releaseURL                      string
+	useEnvoyContrib                 bool
+	healthChecker                   *healthchecker.AggregatedHealthChecker
+	overloadMaxHeapSizeBytes        *uint64
+	overloadMaxActiveConnections    *uint64
 }
 
 // Option is a functional option for ProxyReconciler.
@@ -122,6 +124,20 @@ func WithAggregatedHealthChecker(hc *healthchecker.AggregatedHealthChecker) Opti
 	}
 }
 
+// WithOverloadMaxHeapSizeBytes sets the maximum heap size in bytes for the Envoy overload manager.
+func WithOverloadMaxHeapSizeBytes(size uint64) Option {
+	return func(o *options) {
+		o.overloadMaxHeapSizeBytes = &size
+	}
+}
+
+// WithOverloadMaxActiveConnections sets the maximum number of active downstream connections for the Envoy overload manager.
+func WithOverloadMaxActiveConnections(count uint64) Option {
+	return func(o *options) {
+		o.overloadMaxActiveConnections = &count
+	}
+}
+
 func defaultOptions() *options {
 	return &options{}
 }
@@ -303,6 +319,14 @@ func (r *ProxyReconciler) Reconcile(ctx context.Context, request reconcile.Reque
 			bootstrap.WithXdsServerHost(r.apiServerHost),
 			// TODO(dilyevsky): Add TLS config from r.options.apiServerTLSConfig.
 		}
+
+		if r.options.overloadMaxHeapSizeBytes != nil {
+			bsOpts = append(bsOpts, bootstrap.WithOverloadMaxHeapSizeBytes(*r.options.overloadMaxHeapSizeBytes))
+		}
+
+		if r.options.overloadMaxActiveConnections != nil {
+			bsOpts = append(bsOpts, bootstrap.WithOverloadMaxActiveConnections(*r.options.overloadMaxActiveConnections))
+		}
 		cfg, err := bootstrap.GetRenderedBootstrapConfig(bsOpts...)
 		if err != nil {
 			// If the config is invalid, we can't start the proxy.

pkg/gateway/xds/bootstrap/bootstrap.go

@@ -34,6 +34,9 @@ const (
 	EnvoyReadinessPath    = "/ready"
 )
 
+// defaultEnvoyMaxActiveDownstreamConnections is the default maximum number of active downstream connections.
+var defaultEnvoyMaxActiveDownstreamConnections uint64 = 50000
+
 //go:embed bootstrap.yaml.tpl
 var bootstrapTmplStr string
 
@@ -64,6 +67,15 @@ type bootstrapParameters struct {
 	// StatsMatcher is to control creation of custom Envoy stats with prefix,
 	// suffix, and regex expressions match on the name of the stats.
 	StatsMatcher *StatsMatcherParameters
+	// OverloadManager defines the configuration of the Envoy overload manager.
+	OverloadManager OverloadManagerParameters
+}
+
+type OverloadManagerParameters struct {
+	// MaxHeapSizeBytes defines the maximum heap size in bytes.
+	MaxHeapSizeBytes *uint64
+	// MaxActiveDownstreamConnections defines the maximum number of active downstream connections.
+	MaxActiveDownstreamConnections *uint64
 }
 
 type xdsServerParameters struct {
@@ -121,12 +133,24 @@ type BootstrapConfig struct {
 	XdsServerHost string
 	// XdsServerPort is the port of the Xds Server within Envoy Gateway.
 	XdsServerPort int32
+	// OverloadMaxHeapSizeBytes defines the maximum heap size in bytes for the Envoy overload manager.
+	OverloadMaxHeapSizeBytes *uint64
+	// OverloadMaxActiveDownstreamConnections defines the maximum number of active downstream connections for the Envoy overload manager.
+	OverloadMaxActiveDownstreamConnections *uint64
 }
 
 func defaultBootstrapConfig() *BootstrapConfig {
+	// Create a variable from the constant so we can take its address
+	defaultConnections := defaultEnvoyMaxActiveDownstreamConnections
+
+	// Get the default max heap size from the detector
+	maxHeapSize := GetDefaultMaxHeapSizeBytes()
+
 	return &BootstrapConfig{
-		XdsServerHost: envoyGatewayXdsServerHost,
-		XdsServerPort: DefaultXdsServerPort,
+		XdsServerHost:                          envoyGatewayXdsServerHost,
+		XdsServerPort:                          DefaultXdsServerPort,
+		OverloadMaxHeapSizeBytes:               maxHeapSize,
+		OverloadMaxActiveDownstreamConnections: &defaultConnections,
 	}
 }
 
@@ -148,6 +172,20 @@ func WithXdsServerPort(port int32) BootstrapOption {
 	}
 }
 
+// WithOverloadMaxHeapSizeBytes sets the maximum heap size in bytes for the Envoy overload manager.
+func WithOverloadMaxHeapSizeBytes(size uint64) BootstrapOption {
+	return func(cfg *BootstrapConfig) {
+		cfg.OverloadMaxHeapSizeBytes = &size
+	}
+}
+
+// WithOverloadMaxActiveConnections sets the maximum number of active downstream connections for the Envoy overload manager.
+func WithOverloadMaxActiveConnections(count uint64) BootstrapOption {
+	return func(cfg *BootstrapConfig) {
+		cfg.OverloadMaxActiveDownstreamConnections = &count
+	}
+}
+
 // GetRenderedBootstrapConfig renders the bootstrap YAML string.
 func GetRenderedBootstrapConfig(opts ...BootstrapOption) (string, error) {
 	sOpts := defaultBootstrapConfig()
@@ -170,6 +208,10 @@ func GetRenderedBootstrapConfig(opts ...BootstrapOption) (string, error) {
 				Port:          EnvoyReadinessPort,
 				ReadinessPath: EnvoyReadinessPath,
 			},
+			OverloadManager: OverloadManagerParameters{
+				MaxHeapSizeBytes:               sOpts.OverloadMaxHeapSizeBytes,
+				MaxActiveDownstreamConnections: sOpts.OverloadMaxActiveDownstreamConnections,
+			},
 		},
 	}
 

pkg/gateway/xds/bootstrap/bootstrap.yaml.tpl

@@ -160,3 +160,27 @@ static_resources:
             connection_keepalive:
               interval: 30s
               timeout: 5s
+overload_manager:
+  refresh_interval: 0.25s
+  resource_monitors:
+  - name: "envoy.resource_monitors.global_downstream_max_connections"
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig
+      max_active_downstream_connections: {{ .OverloadManager.MaxActiveDownstreamConnections }}
+  {{- with .OverloadManager.MaxHeapSizeBytes }}
+  - name: "envoy.resource_monitors.fixed_heap"
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig
+      max_heap_size_bytes: {{ . }}
+  actions:
+  - name: "envoy.overload_actions.shrink_heap"
+    triggers:
+    - name: "envoy.resource_monitors.fixed_heap"
+      threshold:
+        value: 0.95
+  - name: "envoy.overload_actions.stop_accepting_requests"
+    triggers:
+    - name: "envoy.resource_monitors.fixed_heap"
+      threshold:
+        value: 0.98
+  {{- end }}

pkg/gateway/xds/bootstrap/bootstrap_test.go

@@ -1,8 +1,3 @@
-// Copyright Envoy Gateway Authors
-// SPDX-License-Identifier: Apache-2.0
-// The full text of the Apache license is available in the LICENSE file at
-// the root of the repo.
-
 package bootstrap
 
 import (
@@ -13,79 +8,45 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"k8s.io/utils/ptr"
-
-	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 )
 
 func TestGetRenderedBootstrapConfig(t *testing.T) {
+	origDetector := defaultDetector
+	defer func() { defaultDetector = origDetector }()
+
+	mockReader := &MockFileReader{
+		files: map[string][]byte{
+			"/sys/fs/cgroup/memory.max": []byte("2147483648\n"),
+		},
+	}
+	mockDetector := NewCgroupMemoryDetector()
+	mockDetector.fileReader = mockReader
+	defaultDetector = mockDetector
+	result := GetCgroupMemoryLimit()
+	assert.Equal(t, uint64(2147483648), result)
+
 	cases := []struct {
-		name         string
-		proxyMetrics *egv1a1.ProxyMetrics
+		name            string
+		overrideOptions []BootstrapOption
 	}{
 		{
-			name: "disable-prometheus",
-			proxyMetrics: &egv1a1.ProxyMetrics{
-				Prometheus: &egv1a1.ProxyPrometheusProvider{
-					Disable: true,
-				},
-			},
-		},
-		{
-			name: "enable-prometheus",
-			proxyMetrics: &egv1a1.ProxyMetrics{
-				Prometheus: &egv1a1.ProxyPrometheusProvider{},
-			},
-		},
-		{
-			name: "otel-metrics",
-			proxyMetrics: &egv1a1.ProxyMetrics{
-				Prometheus: &egv1a1.ProxyPrometheusProvider{
-					Disable: true,
-				},
-				Sinks: []egv1a1.ProxyMetricSink{
-					{
-						Type: egv1a1.MetricSinkTypeOpenTelemetry,
-						OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{
-							Host: "otel-collector.monitoring.svc",
-							Port: 4317,
-						},
-					},
-				},
+			name: "overload-manager",
+			overrideOptions: []BootstrapOption{
+				WithOverloadMaxHeapSizeBytes(1073741824), // 1GB
+				WithOverloadMaxActiveConnections(50000),
 			},
 		},
 		{
-			name: "custom-stats-matcher",
-			proxyMetrics: &egv1a1.ProxyMetrics{
-				Matches: []egv1a1.StringMatch{
-					{
-						Type:  ptr.To(egv1a1.StringMatchExact),
-						Value: "http.foo.bar.cluster.upstream_rq",
-					},
-					{
-						Type:  ptr.To(egv1a1.StringMatchPrefix),
-						Value: "http",
-					},
-					{
-						Type:  ptr.To(egv1a1.StringMatchSuffix),
-						Value: "upstream_rq",
-					},
-					{
-						Type:  ptr.To(egv1a1.StringMatchRegularExpression),
-						Value: "virtual.*",
-					},
-					{
-						Type:  ptr.To(egv1a1.StringMatchPrefix),
-						Value: "cluster",
-					},
-				},
+			name: "overload-manager-cgroup",
+			overrideOptions: []BootstrapOption{
+				WithOverloadMaxActiveConnections(50000),
 			},
 		},
 	}
 
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			got, err := GetRenderedBootstrapConfig(tc.proxyMetrics)
+			got, err := GetRenderedBootstrapConfig(tc.overrideOptions...)
 			require.NoError(t, err)
 
 			if *overrideTestData {