diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index ca03cb5450f..875ce07eac8 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -51213,7 +51213,7 @@ func schema_openshift_api_operator_v1_IngressControllerSpec(ref common.Reference }, "idleConnectionTerminationPolicy": { SchemaProps: spec.SchemaProps{ - Description: "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Immediate\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires\n (300 seconds in OpenShift's configuration, not\n configurable).\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", + Description: "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Immediate\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires.\n By default this is 300 seconds, but it can be changed\n using httpKeepAliveTimeout tuning option.\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", Default: "Immediate", Type: []string{"string"}, Format: "", @@ -51383,6 +51383,12 @@ func schema_openshift_api_operator_v1_IngressControllerTuningOptions(ref common. Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, + "httpKeepAliveTimeout": { + SchemaProps: spec.SchemaProps{ + Description: "httpKeepAliveTimeout defines the maximum allowed time to wait for a new HTTP request to appear on a connection from the client to the router.\n\nThis field expects an unsigned duration string of a decimal number, with optional fraction and a unit suffix, e.g. \"300ms\", \"1.5s\" or \"2m45s\". Valid time units are \"ms\", \"s\", \"m\". The allowed range is from 1 millisecond to 15 minutes.\n\nWhen omitted, this means the user has no opinion and the platform is left to choose a reasonable default. This default is subject to change over time. The current default is 300s.\n\nLow values (tens of milliseconds or less) can cause clients to close and reopen connections for each request, leading to reduced connection sharing. For HTTP/2, special care should be taken with low values. A few seconds is a reasonable starting point to avoid holding idle connections open while still allowing subsequent requests to reuse the connection.\n\nHigh values (minutes or more) favor connection reuse but may cause idle connections to linger longer.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, "tlsInspectDelay": { SchemaProps: spec.SchemaProps{ Description: "tlsInspectDelay defines how long the router can hold data to find a matching route.\n\nSetting this too short can cause the router to fall back to the default certificate for edge-terminated or reencrypt routes even when a better matching certificate could be used.\n\nIf unset, the default inspect delay is 5s", diff --git a/openapi/openapi.json b/openapi/openapi.json index 42ee6c97da5..dc89c1ee100 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -29693,7 +29693,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1.IngressControllerHTTPHeaders" }, "idleConnectionTerminationPolicy": { - "description": "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Immediate\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires\n (300 seconds in OpenShift's configuration, not\n configurable).\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", + "description": "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Immediate\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires.\n By default this is 300 seconds, but it can be changed\n using httpKeepAliveTimeout tuning option.\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", "type": "string", "default": "Immediate" }, @@ -29822,6 +29822,10 @@ "description": "healthCheckInterval defines how long the router waits between two consecutive health checks on its configured backends. This value is applied globally as a default for all routes, but may be overridden per-route by the route annotation \"router.openshift.io/haproxy.health.check.interval\".\n\nExpects an unsigned duration string of decimal numbers, each with optional fraction and a unit suffix, eg \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\" U+00B5 or \"μs\" U+03BC), \"ms\", \"s\", \"m\", \"h\".\n\nSetting this to less than 5s can cause excess traffic due to too frequent TCP health checks and accompanying SYN packet storms. Alternatively, setting this too high can result in increased latency, due to backend servers that are no longer available, but haven't yet been detected as such.\n\nAn empty or zero healthCheckInterval means no opinion and IngressController chooses a default, which is subject to change over time. Currently the default healthCheckInterval value is 5s.\n\nCurrently the minimum allowed value is 1s and the maximum allowed value is 2147483647ms (24.85 days). Both are subject to change over time.", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" }, + "httpKeepAliveTimeout": { + "description": "httpKeepAliveTimeout defines the maximum allowed time to wait for a new HTTP request to appear on a connection from the client to the router.\n\nThis field expects an unsigned duration string of a decimal number, with optional fraction and a unit suffix, e.g. \"300ms\", \"1.5s\" or \"2m45s\". Valid time units are \"ms\", \"s\", \"m\". The allowed range is from 1 millisecond to 15 minutes.\n\nWhen omitted, this means the user has no opinion and the platform is left to choose a reasonable default. This default is subject to change over time. The current default is 300s.\n\nLow values (tens of milliseconds or less) can cause clients to close and reopen connections for each request, leading to reduced connection sharing. For HTTP/2, special care should be taken with low values. A few seconds is a reasonable starting point to avoid holding idle connections open while still allowing subsequent requests to reuse the connection.\n\nHigh values (minutes or more) favor connection reuse but may cause idle connections to linger longer.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + }, "maxConnections": { "description": "maxConnections defines the maximum number of simultaneous connections that can be established per HAProxy process. Increasing this value allows each ingress controller pod to handle more connections but at the cost of additional system resources being consumed.\n\nPermitted values are: empty, 0, -1, and the range 2000-2000000.\n\nIf this field is empty or 0, the IngressController will use the default value of 50000, but the default is subject to change in future releases.\n\nIf the value is -1 then HAProxy will dynamically compute a maximum value based on the available ulimits in the running container. Selecting -1 (i.e., auto) will result in a large value being computed (~520000 on OpenShift >=4.10 clusters) and therefore each HAProxy process will incur significant memory usage compared to the current default of 50000.\n\nSetting a value that is greater than the current operating system limit will prevent the HAProxy process from starting.\n\nIf you choose a discrete value (e.g., 750000) and the router pod is migrated to a new node, there's no guarantee that that new node has identical ulimits configured. In such a scenario the pod would fail to start. If you have nodes with different ulimits configured (e.g., different tuned profiles) and you choose a discrete value then the guidance is to use -1 and let the value be computed dynamically at runtime.\n\nYou can monitor memory usage for router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}'.\n\nYou can monitor memory usage of individual HAProxy processes in router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}/container_processes{container=\"router\",namespace=\"openshift-ingress\"}'.", "type": "integer", diff --git a/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml b/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml index 527eff466bf..1f09bccf3c9 100644 --- a/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml @@ -565,3 +565,110 @@ tests: tuningOptions: connectTimeout: "4 s" expectedError: "IngressController.operator.openshift.io \"default\" is invalid: spec.tuningOptions.connectTimeout: Invalid value: \"4 s\": spec.tuningOptions.connectTimeout in body should match '^(0|([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$'" + - name: Should be able to create an IngressController with valid nominal httpKeepAlive timeout + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: 10s + expected: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Immediate + tuningOptions: + httpKeepAliveTimeout: 10s + - name: Should be able to create an IngressController with valid composite httpKeepAlive timeout + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: 100s300ms + expected: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Immediate + tuningOptions: + httpKeepAliveTimeout: 100s300ms + - name: Should be able to create an IngressController with valid fraction httpKeepAlive timeout + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: 1.5m + expected: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Immediate + tuningOptions: + httpKeepAliveTimeout: 1.5m + - name: Should not be able to create an IngressController with invalid unit httpKeepAlive timeout + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: 3d + expectedError: "IngressController.operator.openshift.io \"default\" is invalid: spec.tuningOptions.httpKeepAliveTimeout: Invalid value: \"string\": httpKeepAliveTimeout must be a valid duration string composed of an unsigned integer value, optionally followed by a decimal fraction and a unit suffix (ms, s, m)" + - name: Should not be able to create an IngressController with invalid space httpKeepAlive timeout + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: "4 s" + expectedError: "IngressController.operator.openshift.io \"default\" is invalid: spec.tuningOptions.httpKeepAliveTimeout: Invalid value: \"string\": httpKeepAliveTimeout must be a valid duration string composed of an unsigned integer value, optionally followed by a decimal fraction and a unit suffix (ms, s, m)" + - name: Should not be able to create an IngressController with httpKeepAlive timeout too big + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: 100m + expectedError: "IngressController.operator.openshift.io \"default\" is invalid: spec.tuningOptions.httpKeepAliveTimeout: Invalid value: \"string\": httpKeepAliveTimeout must be less than or equal to 15 minutes" + - name: Should not be able to create an IngressController with httpKeepAlive timeout too small + initial: | + apiVersion: operator.openshift.io/v1 + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + tuningOptions: + httpKeepAliveTimeout: 0.001ms + expectedError: "IngressController.operator.openshift.io \"default\" is invalid: spec.tuningOptions.httpKeepAliveTimeout: Invalid value: \"string\": httpKeepAliveTimeout must be greater than or equal to 1 millisecond" diff --git a/operator/v1/types_ingress.go b/operator/v1/types_ingress.go index 35b50a8fbd4..e26e3da35ec 100644 --- a/operator/v1/types_ingress.go +++ b/operator/v1/types_ingress.go @@ -281,9 +281,9 @@ type IngressControllerSpec struct { // case HAProxy handles it in the old process and closes // the connection after sending the response. // - // - HAProxy's `timeout http-keep-alive` duration expires - // (300 seconds in OpenShift's configuration, not - // configurable). + // - HAProxy's `timeout http-keep-alive` duration expires. + // By default this is 300 seconds, but it can be changed + // using httpKeepAliveTimeout tuning option. // // - The client's keep-alive timeout expires, causing the // client to close the connection. @@ -1870,6 +1870,36 @@ type IngressControllerTuningOptions struct { // +optional ConnectTimeout *metav1.Duration `json:"connectTimeout,omitempty"` + // httpKeepAliveTimeout defines the maximum allowed time to wait for + // a new HTTP request to appear on a connection from the client to the router. + // + // This field expects an unsigned duration string of a decimal number, with optional + // fraction and a unit suffix, e.g. "300ms", "1.5s" or "2m45s". + // Valid time units are "ms", "s", "m". + // The allowed range is from 1 millisecond to 15 minutes. + // + // When omitted, this means the user has no opinion and the platform is left + // to choose a reasonable default. This default is subject to change over time. + // The current default is 300s. + // + // Low values (tens of milliseconds or less) can cause clients to close and reopen connections + // for each request, leading to reduced connection sharing. + // For HTTP/2, special care should be taken with low values. + // A few seconds is a reasonable starting point to avoid holding idle connections open + // while still allowing subsequent requests to reuse the connection. + // + // High values (minutes or more) favor connection reuse but may cause idle + // connections to linger longer. + // + // +kubebuilder:validation:Type:=string + // +kubebuilder:validation:XValidation:rule="self.matches('^([0-9]+(\\\\.[0-9]+)?(ms|s|m))+$')",message="httpKeepAliveTimeout must be a valid duration string composed of an unsigned integer value, optionally followed by a decimal fraction and a unit suffix (ms, s, m)" + // +kubebuilder:validation:XValidation:rule="!self.matches('^([0-9]+(\\\\.[0-9]+)?(ms|s|m))+$') || duration(self) <= duration('15m')",message="httpKeepAliveTimeout must be less than or equal to 15 minutes" + // +kubebuilder:validation:XValidation:rule="!self.matches('^([0-9]+(\\\\.[0-9]+)?(ms|s|m))+$') || duration(self) >= duration('1ms')",message="httpKeepAliveTimeout must be greater than or equal to 1 millisecond" + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=16 + // +optional + HTTPKeepAliveTimeout *metav1.Duration `json:"httpKeepAliveTimeout,omitempty"` + // tlsInspectDelay defines how long the router can hold data to find a // matching route. // diff --git a/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml b/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml index 10ca42895c3..ee461882aff 100644 --- a/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml +++ b/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml @@ -1280,9 +1280,9 @@ spec: case HAProxy handles it in the old process and closes the connection after sending the response. - - HAProxy's `timeout http-keep-alive` duration expires - (300 seconds in OpenShift's configuration, not - configurable). + - HAProxy's `timeout http-keep-alive` duration expires. + By default this is 300 seconds, but it can be changed + using httpKeepAliveTimeout tuning option. - The client's keep-alive timeout expires, causing the client to close the connection. @@ -2230,6 +2230,44 @@ spec: 2147483647ms (24.85 days). Both are subject to change over time. pattern: ^(0|([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$ type: string + httpKeepAliveTimeout: + description: |- + httpKeepAliveTimeout defines the maximum allowed time to wait for + a new HTTP request to appear on a connection from the client to the router. + + This field expects an unsigned duration string of a decimal number, with optional + fraction and a unit suffix, e.g. "300ms", "1.5s" or "2m45s". + Valid time units are "ms", "s", "m". + The allowed range is from 1 millisecond to 15 minutes. + + When omitted, this means the user has no opinion and the platform is left + to choose a reasonable default. This default is subject to change over time. + The current default is 300s. + + Low values (tens of milliseconds or less) can cause clients to close and reopen connections + for each request, leading to reduced connection sharing. + For HTTP/2, special care should be taken with low values. + A few seconds is a reasonable starting point to avoid holding idle connections open + while still allowing subsequent requests to reuse the connection. + + High values (minutes or more) favor connection reuse but may cause idle + connections to linger longer. + maxLength: 16 + minLength: 1 + type: string + x-kubernetes-validations: + - message: httpKeepAliveTimeout must be a valid duration string + composed of an unsigned integer value, optionally followed + by a decimal fraction and a unit suffix (ms, s, m) + rule: self.matches('^([0-9]+(\\.[0-9]+)?(ms|s|m))+$') + - message: httpKeepAliveTimeout must be less than or equal to + 15 minutes + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) <= duration(''15m'')' + - message: httpKeepAliveTimeout must be greater than or equal + to 1 millisecond + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) >= duration(''1ms'')' maxConnections: description: |- maxConnections defines the maximum number of simultaneous diff --git a/operator/v1/zz_generated.deepcopy.go b/operator/v1/zz_generated.deepcopy.go index d2e74e68387..1bab9f58034 100644 --- a/operator/v1/zz_generated.deepcopy.go +++ b/operator/v1/zz_generated.deepcopy.go @@ -2497,6 +2497,11 @@ func (in *IngressControllerTuningOptions) DeepCopyInto(out *IngressControllerTun *out = new(metav1.Duration) **out = **in } + if in.HTTPKeepAliveTimeout != nil { + in, out := &in.HTTPKeepAliveTimeout, &out.HTTPKeepAliveTimeout + *out = new(metav1.Duration) + **out = **in + } if in.TLSInspectDelay != nil { in, out := &in.TLSInspectDelay, &out.TLSInspectDelay *out = new(metav1.Duration) diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml index ebec90a0b4e..e48c649f6fb 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml @@ -1074,9 +1074,9 @@ spec: case HAProxy handles it in the old process and closes the connection after sending the response. - - HAProxy's `timeout http-keep-alive` duration expires - (300 seconds in OpenShift's configuration, not - configurable). + - HAProxy's `timeout http-keep-alive` duration expires. + By default this is 300 seconds, but it can be changed + using httpKeepAliveTimeout tuning option. - The client's keep-alive timeout expires, causing the client to close the connection. @@ -2006,6 +2006,44 @@ spec: 2147483647ms (24.85 days). Both are subject to change over time. pattern: ^(0|([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$ type: string + httpKeepAliveTimeout: + description: |- + httpKeepAliveTimeout defines the maximum allowed time to wait for + a new HTTP request to appear on a connection from the client to the router. + + This field expects an unsigned duration string of a decimal number, with optional + fraction and a unit suffix, e.g. "300ms", "1.5s" or "2m45s". + Valid time units are "ms", "s", "m". + The allowed range is from 1 millisecond to 15 minutes. + + When omitted, this means the user has no opinion and the platform is left + to choose a reasonable default. This default is subject to change over time. + The current default is 300s. + + Low values (tens of milliseconds or less) can cause clients to close and reopen connections + for each request, leading to reduced connection sharing. + For HTTP/2, special care should be taken with low values. + A few seconds is a reasonable starting point to avoid holding idle connections open + while still allowing subsequent requests to reuse the connection. + + High values (minutes or more) favor connection reuse but may cause idle + connections to linger longer. + maxLength: 16 + minLength: 1 + type: string + x-kubernetes-validations: + - message: httpKeepAliveTimeout must be a valid duration string + composed of an unsigned integer value, optionally followed + by a decimal fraction and a unit suffix (ms, s, m) + rule: self.matches('^([0-9]+(\\.[0-9]+)?(ms|s|m))+$') + - message: httpKeepAliveTimeout must be less than or equal to + 15 minutes + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) <= duration(''15m'')' + - message: httpKeepAliveTimeout must be greater than or equal + to 1 millisecond + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) >= duration(''1ms'')' maxConnections: description: |- maxConnections defines the maximum number of simultaneous diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml index 10a5b8a259e..29d818f9949 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml @@ -1223,9 +1223,9 @@ spec: case HAProxy handles it in the old process and closes the connection after sending the response. - - HAProxy's `timeout http-keep-alive` duration expires - (300 seconds in OpenShift's configuration, not - configurable). + - HAProxy's `timeout http-keep-alive` duration expires. + By default this is 300 seconds, but it can be changed + using httpKeepAliveTimeout tuning option. - The client's keep-alive timeout expires, causing the client to close the connection. @@ -2155,6 +2155,44 @@ spec: 2147483647ms (24.85 days). Both are subject to change over time. pattern: ^(0|([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$ type: string + httpKeepAliveTimeout: + description: |- + httpKeepAliveTimeout defines the maximum allowed time to wait for + a new HTTP request to appear on a connection from the client to the router. + + This field expects an unsigned duration string of a decimal number, with optional + fraction and a unit suffix, e.g. "300ms", "1.5s" or "2m45s". + Valid time units are "ms", "s", "m". + The allowed range is from 1 millisecond to 15 minutes. + + When omitted, this means the user has no opinion and the platform is left + to choose a reasonable default. This default is subject to change over time. + The current default is 300s. + + Low values (tens of milliseconds or less) can cause clients to close and reopen connections + for each request, leading to reduced connection sharing. + For HTTP/2, special care should be taken with low values. + A few seconds is a reasonable starting point to avoid holding idle connections open + while still allowing subsequent requests to reuse the connection. + + High values (minutes or more) favor connection reuse but may cause idle + connections to linger longer. + maxLength: 16 + minLength: 1 + type: string + x-kubernetes-validations: + - message: httpKeepAliveTimeout must be a valid duration string + composed of an unsigned integer value, optionally followed + by a decimal fraction and a unit suffix (ms, s, m) + rule: self.matches('^([0-9]+(\\.[0-9]+)?(ms|s|m))+$') + - message: httpKeepAliveTimeout must be less than or equal to + 15 minutes + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) <= duration(''15m'')' + - message: httpKeepAliveTimeout must be greater than or equal + to 1 millisecond + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) >= duration(''1ms'')' maxConnections: description: |- maxConnections defines the maximum number of simultaneous diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController+IngressControllerLBSubnetsAWS.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController+IngressControllerLBSubnetsAWS.yaml index 713c1afbb19..5486d37ffc5 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController+IngressControllerLBSubnetsAWS.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController+IngressControllerLBSubnetsAWS.yaml @@ -1282,9 +1282,9 @@ spec: case HAProxy handles it in the old process and closes the connection after sending the response. - - HAProxy's `timeout http-keep-alive` duration expires - (300 seconds in OpenShift's configuration, not - configurable). + - HAProxy's `timeout http-keep-alive` duration expires. + By default this is 300 seconds, but it can be changed + using httpKeepAliveTimeout tuning option. - The client's keep-alive timeout expires, causing the client to close the connection. @@ -2214,6 +2214,44 @@ spec: 2147483647ms (24.85 days). Both are subject to change over time. pattern: ^(0|([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$ type: string + httpKeepAliveTimeout: + description: |- + httpKeepAliveTimeout defines the maximum allowed time to wait for + a new HTTP request to appear on a connection from the client to the router. + + This field expects an unsigned duration string of a decimal number, with optional + fraction and a unit suffix, e.g. "300ms", "1.5s" or "2m45s". + Valid time units are "ms", "s", "m". + The allowed range is from 1 millisecond to 15 minutes. + + When omitted, this means the user has no opinion and the platform is left + to choose a reasonable default. This default is subject to change over time. + The current default is 300s. + + Low values (tens of milliseconds or less) can cause clients to close and reopen connections + for each request, leading to reduced connection sharing. + For HTTP/2, special care should be taken with low values. + A few seconds is a reasonable starting point to avoid holding idle connections open + while still allowing subsequent requests to reuse the connection. + + High values (minutes or more) favor connection reuse but may cause idle + connections to linger longer. + maxLength: 16 + minLength: 1 + type: string + x-kubernetes-validations: + - message: httpKeepAliveTimeout must be a valid duration string + composed of an unsigned integer value, optionally followed + by a decimal fraction and a unit suffix (ms, s, m) + rule: self.matches('^([0-9]+(\\.[0-9]+)?(ms|s|m))+$') + - message: httpKeepAliveTimeout must be less than or equal to + 15 minutes + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) <= duration(''15m'')' + - message: httpKeepAliveTimeout must be greater than or equal + to 1 millisecond + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) >= duration(''1ms'')' maxConnections: description: |- maxConnections defines the maximum number of simultaneous diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml index 55625cb29ae..b6f78bbafbf 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml @@ -1114,9 +1114,9 @@ spec: case HAProxy handles it in the old process and closes the connection after sending the response. - - HAProxy's `timeout http-keep-alive` duration expires - (300 seconds in OpenShift's configuration, not - configurable). + - HAProxy's `timeout http-keep-alive` duration expires. + By default this is 300 seconds, but it can be changed + using httpKeepAliveTimeout tuning option. - The client's keep-alive timeout expires, causing the client to close the connection. @@ -2046,6 +2046,44 @@ spec: 2147483647ms (24.85 days). Both are subject to change over time. pattern: ^(0|([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$ type: string + httpKeepAliveTimeout: + description: |- + httpKeepAliveTimeout defines the maximum allowed time to wait for + a new HTTP request to appear on a connection from the client to the router. + + This field expects an unsigned duration string of a decimal number, with optional + fraction and a unit suffix, e.g. "300ms", "1.5s" or "2m45s". + Valid time units are "ms", "s", "m". + The allowed range is from 1 millisecond to 15 minutes. + + When omitted, this means the user has no opinion and the platform is left + to choose a reasonable default. This default is subject to change over time. + The current default is 300s. + + Low values (tens of milliseconds or less) can cause clients to close and reopen connections + for each request, leading to reduced connection sharing. + For HTTP/2, special care should be taken with low values. + A few seconds is a reasonable starting point to avoid holding idle connections open + while still allowing subsequent requests to reuse the connection. + + High values (minutes or more) favor connection reuse but may cause idle + connections to linger longer. + maxLength: 16 + minLength: 1 + type: string + x-kubernetes-validations: + - message: httpKeepAliveTimeout must be a valid duration string + composed of an unsigned integer value, optionally followed + by a decimal fraction and a unit suffix (ms, s, m) + rule: self.matches('^([0-9]+(\\.[0-9]+)?(ms|s|m))+$') + - message: httpKeepAliveTimeout must be less than or equal to + 15 minutes + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) <= duration(''15m'')' + - message: httpKeepAliveTimeout must be greater than or equal + to 1 millisecond + rule: '!self.matches(''^([0-9]+(\\.[0-9]+)?(ms|s|m))+$'') || + duration(self) >= duration(''1ms'')' maxConnections: description: |- maxConnections defines the maximum number of simultaneous diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 206dd98c43b..1070eae18cf 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -1080,7 +1080,7 @@ var map_IngressControllerSpec = map[string]string{ "tuningOptions": "tuningOptions defines parameters for adjusting the performance of ingress controller pods. All fields are optional and will use their respective defaults if not set. See specific tuningOptions fields for more details.\n\nSetting fields within tuningOptions is generally not recommended. The default values are suitable for most configurations.", "unsupportedConfigOverrides": "unsupportedConfigOverrides allows specifying unsupported configuration options. Its use is unsupported.", "httpCompression": "httpCompression defines a policy for HTTP traffic compression. By default, there is no HTTP compression.", - "idleConnectionTerminationPolicy": "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Immediate\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires\n (300 seconds in OpenShift's configuration, not\n configurable).\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", + "idleConnectionTerminationPolicy": "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Immediate\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires.\n By default this is 300 seconds, but it can be changed\n using httpKeepAliveTimeout tuning option.\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", } func (IngressControllerSpec) SwaggerDoc() map[string]string { @@ -1115,6 +1115,7 @@ var map_IngressControllerTuningOptions = map[string]string{ "serverFinTimeout": "serverFinTimeout defines how long a connection will be held open while waiting for the server/backend response to the client closing the connection.\n\nIf unset, the default timeout is 1s", "tunnelTimeout": "tunnelTimeout defines how long a tunnel connection (including websockets) will be held open while the tunnel is idle.\n\nIf unset, the default timeout is 1h", "connectTimeout": "connectTimeout defines the maximum time to wait for a connection attempt to a server/backend to succeed.\n\nThis field expects an unsigned duration string of decimal numbers, each with optional fraction and a unit suffix, e.g. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\" U+00B5 or \"μs\" U+03BC), \"ms\", \"s\", \"m\", \"h\".\n\nWhen omitted, this means the user has no opinion and the platform is left to choose a reasonable default. This default is subject to change over time. The current default is 5s.", + "httpKeepAliveTimeout": "httpKeepAliveTimeout defines the maximum allowed time to wait for a new HTTP request to appear on a connection from the client to the router.\n\nThis field expects an unsigned duration string of a decimal number, with optional fraction and a unit suffix, e.g. \"300ms\", \"1.5s\" or \"2m45s\". Valid time units are \"ms\", \"s\", \"m\". The allowed range is from 1 millisecond to 15 minutes.\n\nWhen omitted, this means the user has no opinion and the platform is left to choose a reasonable default. This default is subject to change over time. The current default is 300s.\n\nLow values (tens of milliseconds or less) can cause clients to close and reopen connections for each request, leading to reduced connection sharing. For HTTP/2, special care should be taken with low values. A few seconds is a reasonable starting point to avoid holding idle connections open while still allowing subsequent requests to reuse the connection.\n\nHigh values (minutes or more) favor connection reuse but may cause idle connections to linger longer.", "tlsInspectDelay": "tlsInspectDelay defines how long the router can hold data to find a matching route.\n\nSetting this too short can cause the router to fall back to the default certificate for edge-terminated or reencrypt routes even when a better matching certificate could be used.\n\nIf unset, the default inspect delay is 5s", "healthCheckInterval": "healthCheckInterval defines how long the router waits between two consecutive health checks on its configured backends. This value is applied globally as a default for all routes, but may be overridden per-route by the route annotation \"router.openshift.io/haproxy.health.check.interval\".\n\nExpects an unsigned duration string of decimal numbers, each with optional fraction and a unit suffix, eg \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\" U+00B5 or \"μs\" U+03BC), \"ms\", \"s\", \"m\", \"h\".\n\nSetting this to less than 5s can cause excess traffic due to too frequent TCP health checks and accompanying SYN packet storms. Alternatively, setting this too high can result in increased latency, due to backend servers that are no longer available, but haven't yet been detected as such.\n\nAn empty or zero healthCheckInterval means no opinion and IngressController chooses a default, which is subject to change over time. Currently the default healthCheckInterval value is 5s.\n\nCurrently the minimum allowed value is 1s and the maximum allowed value is 2147483647ms (24.85 days). Both are subject to change over time.", "maxConnections": "maxConnections defines the maximum number of simultaneous connections that can be established per HAProxy process. Increasing this value allows each ingress controller pod to handle more connections but at the cost of additional system resources being consumed.\n\nPermitted values are: empty, 0, -1, and the range 2000-2000000.\n\nIf this field is empty or 0, the IngressController will use the default value of 50000, but the default is subject to change in future releases.\n\nIf the value is -1 then HAProxy will dynamically compute a maximum value based on the available ulimits in the running container. Selecting -1 (i.e., auto) will result in a large value being computed (~520000 on OpenShift >=4.10 clusters) and therefore each HAProxy process will incur significant memory usage compared to the current default of 50000.\n\nSetting a value that is greater than the current operating system limit will prevent the HAProxy process from starting.\n\nIf you choose a discrete value (e.g., 750000) and the router pod is migrated to a new node, there's no guarantee that that new node has identical ulimits configured. In such a scenario the pod would fail to start. If you have nodes with different ulimits configured (e.g., different tuned profiles) and you choose a discrete value then the guidance is to use -1 and let the value be computed dynamically at runtime.\n\nYou can monitor memory usage for router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}'.\n\nYou can monitor memory usage of individual HAProxy processes in router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}/container_processes{container=\"router\",namespace=\"openshift-ingress\"}'.",