UPSTREAM: <carry>: send Retry-After when not ready with a caller opt in

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -893,6 +893,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 	if c.ShutdownSendRetryAfter {
 		handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.AfterShutdownDelayDuration.Signaled())
 	}
+	handler = genericfilters.WithOptInRetryAfter(handler, c.newNotReadyRetryAfterFunc())
 	handler = genericfilters.WithHTTPLogging(handler, c.newIsTerminatingFunc())
 	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
 		handler = genericapifilters.WithTracing(handler, c.TracerProvider)

staging/src/k8s.io/apiserver/pkg/server/filters/patch_optin_retry.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+)
+
+func WithOptInRetryAfter(handler http.Handler, notReadyRetryAfterFn ShouldRespondWithRetryAfterFunc) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		var (
+			params         *RetryAfterParams
+			sendRetryAfter bool
+		)
+		if value := req.Header.Get("X-OpenShift-Internal-If-Not-Ready"); value == "reject" {
+			// the caller opted in for the request to be rejected if the server is not ready
+			params, sendRetryAfter = notReadyRetryAfterFn()
+		}
+
+		if !sendRetryAfter {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		// If we are here this means it's time to send Retry-After response
+		//
+		// Copied from net/http2 library
+		// "Connection" headers aren't allowed in HTTP/2 (RFC 7540, 8.1.2.2),
+		// but respect "Connection" == "close" to mean sending a GOAWAY and tearing
+		// down the TCP connection notReadyRetryAfterFn idle, like we do for HTTP/1.
+		if params.TearDownConnection {
+			w.Header().Set("Connection", "close")
+		}
+
+		// Return a 429 status asking the client to try again after 5 seconds
+		w.Header().Set("Retry-After", "5")
+		http.Error(w, params.Message, http.StatusTooManyRequests)
+	})
+}

staging/src/k8s.io/apiserver/pkg/server/patch_config.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package server
 
+import (
+	genericfilters "k8s.io/apiserver/pkg/server/filters"
+)
+
 // newIsTerminatingFunc returns a 'func() bool' that relies on the
 // 'ShutdownInitiated' life cycle signal of answer if the apiserver
 // has started the termination process.
@@ -37,3 +41,18 @@ func (c *Config) newIsTerminatingFunc() func() bool {
 		}
 	}
 }
+
+func (c *Config) newNotReadyRetryAfterFunc() genericfilters.ShouldRespondWithRetryAfterFunc {
+	params := &genericfilters.RetryAfterParams{
+		Message: "The apiserver hasn't been fully initialized yet, please try again later.",
+	}
+
+	return func() (*genericfilters.RetryAfterParams, bool) {
+		select {
+		case <-c.lifecycleSignals.HasBeenReady.Signaled():
+			return nil, false
+		default:
+			return params, true
+		}
+	}
+}