expose configuration for envoy's RateLimitedAsResourceExhausted (proj…

…ectcontour#4971)

The Rate Limit filter in Envoy translates a 429 HTTP response code
to UNAVAILABLE as specified in the gRPC mapping document, but Google recommends
translating it to RESOURCE_EXHAUSTED
(see https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md)

This commit introduces a new setting to allow contour to forward the same parameter
introduced in envoyproxy/envoy#4879

The default value is disabled to retain the original behaviour of returning UNAVAILABLE,
as changing it would be a breaking change.

Closes projectcontour#4901.

Signed-off-by: Víctor Roldán Betancort <vroldanbet@authzed.com>
Signed-off-by: yy <yang.yang@daocloud.io>

apis/projectcontour/v1alpha1/contourconfig.go

@@ -621,6 +621,12 @@ type RateLimitServiceConfig struct {
 	// ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
 	// +optional
 	EnableXRateLimitHeaders *bool `json:"enableXRateLimitHeaders,omitempty"`
+
+	// EnableResourceExhaustedCode enables translating error code 429 to
+	// grpc code RESOURCE_EXHAUSTED. When disabled it's translated to UNAVAILABLE
+	//
+	// +optional
+	EnableResourceExhaustedCode *bool `json:"enableResourceExhaustedCode,omitempty"`
 }
 
 // TracingConfig defines properties for exporting trace data to OpenTelemetry.

changelogs/unreleased/4971-vroldanbet-small.md

@@ -0,0 +1 @@
+expose configuration for envoy's RateLimitedAsResourceExhausted

cmd/contour/serve.go

@@ -668,12 +668,13 @@ func (s *Server) setupRateLimitService(contourConfiguration contour_api_v1alpha1
 	}
 
 	return &xdscache_v3.RateLimitConfig{
-		ExtensionService:        key,
-		SNI:                     sni,
-		Domain:                  contourConfiguration.RateLimitService.Domain,
-		Timeout:                 responseTimeout,
-		FailOpen:                ref.Val(contourConfiguration.RateLimitService.FailOpen, false),
-		EnableXRateLimitHeaders: ref.Val(contourConfiguration.RateLimitService.EnableXRateLimitHeaders, false),
+		ExtensionService:            key,
+		SNI:                         sni,
+		Domain:                      contourConfiguration.RateLimitService.Domain,
+		Timeout:                     responseTimeout,
+		FailOpen:                    ref.Val(contourConfiguration.RateLimitService.FailOpen, false),
+		EnableXRateLimitHeaders:     ref.Val(contourConfiguration.RateLimitService.EnableXRateLimitHeaders, false),
+		EnableResourceExhaustedCode: ref.Val(contourConfiguration.RateLimitService.EnableResourceExhaustedCode, false),
 	}, nil
 }
 

cmd/contour/servecontext.go

@@ -395,9 +395,10 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 				Name:      nsedName.Name,
 				Namespace: nsedName.Namespace,
 			},
-			Domain:                  ctx.Config.RateLimitService.Domain,
-			FailOpen:                ref.To(ctx.Config.RateLimitService.FailOpen),
-			EnableXRateLimitHeaders: ref.To(ctx.Config.RateLimitService.EnableXRateLimitHeaders),
+			Domain:                      ctx.Config.RateLimitService.Domain,
+			FailOpen:                    ref.To(ctx.Config.RateLimitService.FailOpen),
+			EnableXRateLimitHeaders:     ref.To(ctx.Config.RateLimitService.EnableXRateLimitHeaders),
+			EnableResourceExhaustedCode: ref.To(ctx.Config.RateLimitService.EnableResourceExhaustedCode),
 		}
 	}
 

cmd/contour/servecontext_test.go

@@ -623,10 +623,11 @@ func TestConvertServeContext(t *testing.T) {
 		"ratelimit": {
 			getServeContext: func(ctx *serveContext) *serveContext {
 				ctx.Config.RateLimitService = config.RateLimitService{
-					ExtensionService:        "ratens/ratelimitext",
-					Domain:                  "contour",
-					FailOpen:                true,
-					EnableXRateLimitHeaders: true,
+					ExtensionService:            "ratens/ratelimitext",
+					Domain:                      "contour",
+					FailOpen:                    true,
+					EnableXRateLimitHeaders:     true,
+					EnableResourceExhaustedCode: true,
 				}
 				return ctx
 			},
@@ -636,9 +637,10 @@ func TestConvertServeContext(t *testing.T) {
 						Name:      "ratelimitext",
 						Namespace: "ratens",
 					},
-					Domain:                  "contour",
-					FailOpen:                ref.To(true),
-					EnableXRateLimitHeaders: ref.To(true),
+					Domain:                      "contour",
+					FailOpen:                    ref.To(true),
+					EnableXRateLimitHeaders:     ref.To(true),
+					EnableResourceExhaustedCode: ref.To(true),
 				}
 				return cfg
 			},

examples/contour/01-contour-config.yaml

@@ -145,6 +145,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:

examples/contour/01-crds.yaml

@@ -521,6 +521,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3521,6 +3526,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

examples/render/contour-deployment.yaml

@@ -178,6 +178,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:
@@ -731,6 +734,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3731,6 +3739,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

examples/render/contour-gateway-provisioner.yaml

@@ -535,6 +535,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3535,6 +3540,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

examples/render/contour-gateway.yaml

@@ -184,6 +184,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:
@@ -737,6 +740,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3737,6 +3745,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

examples/render/contour.yaml

@@ -178,6 +178,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:
@@ -731,6 +734,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3731,6 +3739,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

internal/envoy/v3/ratelimit.go

@@ -123,12 +123,13 @@ func GlobalRateLimits(descriptors []*dag.RateLimitDescriptor) []*envoy_route_v3.
 // GlobalRateLimitConfig stores configuration for
 // an HTTP global rate limiting filter.
 type GlobalRateLimitConfig struct {
-	ExtensionService        types.NamespacedName
-	SNI                     string
-	FailOpen                bool
-	Timeout                 timeout.Setting
-	Domain                  string
-	EnableXRateLimitHeaders bool
+	ExtensionService            types.NamespacedName
+	SNI                         string
+	FailOpen                    bool
+	Timeout                     timeout.Setting
+	Domain                      string
+	EnableXRateLimitHeaders     bool
+	EnableResourceExhaustedCode bool
 }
 
 // GlobalRateLimitFilter returns a configured HTTP global rate limit filter,
@@ -149,7 +150,8 @@ func GlobalRateLimitFilter(config *GlobalRateLimitConfig) *http.HttpFilter {
 					GrpcService:         GrpcService(dag.ExtensionClusterName(config.ExtensionService), config.SNI, timeout.DefaultSetting()),
 					TransportApiVersion: envoy_core_v3.ApiVersion_V3,
 				},
-				EnableXRatelimitHeaders: enableXRateLimitHeaders(config.EnableXRateLimitHeaders),
+				EnableXRatelimitHeaders:        enableXRateLimitHeaders(config.EnableXRateLimitHeaders),
+				RateLimitedAsResourceExhausted: config.EnableResourceExhaustedCode,
 			}),
 		},
 	}

internal/envoy/v3/ratelimit_test.go

@@ -372,6 +372,37 @@ func TestGlobalRateLimitFilter(t *testing.T) {
 				},
 			},
 		},
+		"EnableResourceExhaustedCode=true is configured correctly": {
+			cfg: &GlobalRateLimitConfig{
+				ExtensionService:            k8s.NamespacedNameFrom("projectcontour/ratelimit"),
+				Timeout:                     timeout.DurationSetting(7 * time.Second),
+				Domain:                      "domain",
+				FailOpen:                    true,
+				EnableResourceExhaustedCode: true,
+			},
+			want: &http.HttpFilter{
+				Name: wellknown.HTTPRateLimit,
+				ConfigType: &http.HttpFilter_TypedConfig{
+					TypedConfig: protobuf.MustMarshalAny(&ratelimit_filter_v3.RateLimit{
+						Domain:          "domain",
+						Timeout:         durationpb.New(7 * time.Second),
+						FailureModeDeny: false,
+						RateLimitService: &ratelimit_config_v3.RateLimitServiceConfig{
+							GrpcService: &envoy_core_v3.GrpcService{
+								TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{
+									EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{
+										ClusterName: "extension/projectcontour/ratelimit",
+										Authority:   "extension.projectcontour.ratelimit",
+									},
+								},
+							},
+							TransportApiVersion: envoy_core_v3.ApiVersion_V3,
+						},
+						RateLimitedAsResourceExhausted: true,
+					}),
+				},
+			},
+		},
 	}
 
 	for name, tc := range tests {

internal/xdscache/v3/listener.go

@@ -144,12 +144,13 @@ type ListenerConfig struct {
 }
 
 type RateLimitConfig struct {
-	ExtensionService        types.NamespacedName
-	SNI                     string
-	Domain                  string
-	Timeout                 timeout.Setting
-	FailOpen                bool
-	EnableXRateLimitHeaders bool
+	ExtensionService            types.NamespacedName
+	SNI                         string
+	Domain                      string
+	Timeout                     timeout.Setting
+	FailOpen                    bool
+	EnableXRateLimitHeaders     bool
+	EnableResourceExhaustedCode bool
 }
 
 // DefaultListeners returns the configured Listeners or a single
@@ -569,12 +570,13 @@ func envoyGlobalRateLimitConfig(config *RateLimitConfig) *envoy_v3.GlobalRateLim
 	}
 
 	return &envoy_v3.GlobalRateLimitConfig{
-		ExtensionService:        config.ExtensionService,
-		SNI:                     config.SNI,
-		FailOpen:                config.FailOpen,
-		Timeout:                 config.Timeout,
-		Domain:                  config.Domain,
-		EnableXRateLimitHeaders: config.EnableXRateLimitHeaders,
+		ExtensionService:            config.ExtensionService,
+		SNI:                         config.SNI,
+		FailOpen:                    config.FailOpen,
+		Timeout:                     config.Timeout,
+		Domain:                      config.Domain,
+		EnableXRateLimitHeaders:     config.EnableXRateLimitHeaders,
+		EnableResourceExhaustedCode: config.EnableResourceExhaustedCode,
 	}
 }