From e257d176650ae13393d142ce971b7fc16a99f31a Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Sat, 26 Aug 2017 12:33:33 -0700 Subject: [PATCH 1/2] Add --request-timeout to allow the global request timeout of 60 seconds to be configured. --- cmd/kube-apiserver/app/options/options_test.go | 1 + staging/src/k8s.io/apiserver/pkg/server/config.go | 10 +++++++--- .../k8s.io/apiserver/pkg/server/filters/timeout.go | 12 +++++------- .../pkg/server/options/server_run_options.go | 9 +++++++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 06eee371d50a4..a9aa37bb6e424 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -34,6 +34,7 @@ func TestAddFlagsFlag(t *testing.T) { args := []string{ "--enable-swagger-ui=true", + "--request-timeout=2m", } f.Parse(args) if !s.Features.EnableSwaggerUI { diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 9f8d6ab620749..2642285699f98 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -137,8 +137,11 @@ type Config struct { // RESTOptionsGetter is used to construct RESTStorage types via the generic registry. RESTOptionsGetter genericregistry.RESTOptionsGetter - // If specified, requests will be allocated a random timeout between this value, and twice this value. - // Note that it is up to the request handlers to ignore or honor this timeout. In seconds. + // If specified, all requests except those which match the LongRunningFunc predicate will timeout + // after this duration. + RequestTimeout time.Duration + // If specified, long running requests such as watch will be allocated a random timeout between this value, and + // twice this value. Note that it is up to the request handlers to ignore or honor this timeout. In seconds. MinRequestTimeout int // MaxRequestsInFlight is the maximum number of parallel non-long-running requests. Every further // request has to wait. Applies only to non-mutating requests. @@ -205,6 +208,7 @@ func NewConfig() *Config { EnableProfiling: true, MaxRequestsInFlight: 400, MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(60) * time.Second, MinRequestTimeout: 1800, // Default to treating watch as a long-running operation @@ -430,7 +434,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) (secure, insec generic := func(handler http.Handler) http.Handler { handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") handler = genericfilters.WithPanicRecovery(handler, c.RequestContextMapper) - handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc) + handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc, c.RequestTimeout) handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.RequestContextMapper, c.LongRunningFunc) handler = genericapifilters.WithRequestInfo(handler, NewRequestInfoResolver(c), c.RequestContextMapper) handler = apirequest.WithRequestContext(handler, c.RequestContextMapper) diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go b/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go index ed2de1a997234..93d07cd73136e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go @@ -30,12 +30,10 @@ import ( apirequest "k8s.io/apiserver/pkg/endpoints/request" ) -const globalTimeout = time.Minute - var errConnKilled = fmt.Errorf("kill connection/stream") -// WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by globalTimeout. -func WithTimeoutForNonLongRunningRequests(handler http.Handler, requestContextMapper apirequest.RequestContextMapper, longRunning LongRunningRequestCheck) http.Handler { +// WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout. +func WithTimeoutForNonLongRunningRequests(handler http.Handler, requestContextMapper apirequest.RequestContextMapper, longRunning LongRunningRequestCheck, timeout time.Duration) http.Handler { if longRunning == nil { return handler } @@ -44,19 +42,19 @@ func WithTimeoutForNonLongRunningRequests(handler http.Handler, requestContextMa ctx, ok := requestContextMapper.Get(req) if !ok { // if this happens, the handler chain isn't setup correctly because there is no context mapper - return time.After(globalTimeout), apierrors.NewInternalError(fmt.Errorf("no context found for request during timeout")) + return time.After(timeout), apierrors.NewInternalError(fmt.Errorf("no context found for request during timeout")) } requestInfo, ok := apirequest.RequestInfoFrom(ctx) if !ok { // if this happens, the handler chain isn't setup correctly because there is no request info - return time.After(globalTimeout), apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout")) + return time.After(timeout), apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout")) } if longRunning(req, requestInfo) { return nil, nil } - return time.After(globalTimeout), apierrors.NewServerTimeout(schema.GroupResource{Group: requestInfo.APIGroup, Resource: requestInfo.Resource}, requestInfo.Verb, 0) + return time.After(timeout), apierrors.NewServerTimeout(schema.GroupResource{Group: requestInfo.APIGroup, Resource: requestInfo.Resource}, requestInfo.Verb, 0) } return WithTimeout(handler, timeoutFunc) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index a3733d49b4e50..d7f680acb5ef7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -20,6 +20,7 @@ import ( "fmt" "net" "strings" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/admission" @@ -42,6 +43,7 @@ type ServerRunOptions struct { ExternalHost string MaxRequestsInFlight int MaxMutatingRequestsInFlight int + RequestTimeout time.Duration MinRequestTimeout int TargetRAMMB int WatchCacheSizes []string @@ -54,6 +56,7 @@ func NewServerRunOptions() *ServerRunOptions { AdmissionControl: "AlwaysAdmit", MaxRequestsInFlight: defaults.MaxRequestsInFlight, MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight, + RequestTimeout: defaults.RequestTimeout, MinRequestTimeout: defaults.MinRequestTimeout, } } @@ -64,6 +67,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.ExternalAddress = s.ExternalHost c.MaxRequestsInFlight = s.MaxRequestsInFlight c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight + c.RequestTimeout = s.RequestTimeout c.MinRequestTimeout = s.MinRequestTimeout c.PublicAddress = s.AdvertiseAddress @@ -144,6 +148,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "The maximum number of mutating requests in flight at a given time. When the server exceeds this, "+ "it rejects requests. Zero for no limit.") + fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+ + "An optional field indicating the duration a handler must keep a request open before timing "+ + "it out. This is the default request timeout for requests but may be overridden by flags such as "+ + "--min-request-timeout for specific types of requests.") + fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+ "An optional field indicating the minimum number of seconds a handler must keep "+ "a request open before timing it out. Currently only honored by the watch request "+ From ce881fc0e27d9f3be68aab49fd207f8f00aa8839 Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Sat, 26 Aug 2017 12:33:33 -0700 Subject: [PATCH 2/2] Add --request-timeout to kube-apiserver to make global request timeout configurable. --- staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index 142882a89e861..3709f5bd5a99e 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -194,7 +194,7 @@ func (h *handlerChainConfig) handlerChain(apiHandler http.Handler, c *genericapi handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") handler = genericfilters.WithPanicRecovery(handler, c.RequestContextMapper) - handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc) + handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc, c.RequestTimeout) handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.RequestContextMapper, c.LongRunningFunc) handler = genericapifilters.WithRequestInfo(handler, genericapiserver.NewRequestInfoResolver(c), c.RequestContextMapper) handler = genericapirequest.WithRequestContext(handler, c.RequestContextMapper)