fix: disable HTTP2 connections by default

This change mitigates CVE-2023-44487 by disabling HTTP2 by default and forcing HTTP/1.1 until the Go standard library and golang.org/x/net are fully fixed. Right now, it is possible for authenticated and unauthenticated users to hold open HTTP2 connections and consume huge amounts of memory. It is possible to revert back the change by using the `--web.enable-http2` argument. Before this change: ``` curl -kv https://localhost:8443/metrics * Trying 127.0.0.1:8443... * Connected to localhost (127.0.0.1) port 8443 (#0) * ALPN: offers h2,http/1.1 [...] * ALPN: server accepted h2 [...] * using HTTP/2 * h2h3 [:method: GET] * h2h3 [:path: /metrics] * h2h3 [:scheme: https] * h2h3 [:authority: localhost:8443] * h2h3 [user-agent: curl/8.0.1] * h2h3 [accept: */*] * Using Stream ID: 1 (easy handle 0x5594d4614b10) [...] > GET /metrics HTTP/2 [...] ``` After this change: ``` curl -kv https://localhost:8443/metrics * Trying 127.0.0.1:8443... * Connected to localhost (127.0.0.1) port 8443 (#0) * ALPN: offers h2,http/1.1 [...] * ALPN: server accepted http/1.1 [...] * using HTTP/1.1 > GET /metrics HTTP/1.1 > Host: localhost:8443 > User-Agent: curl/8.0.1 > Accept: */* [...] < HTTP/1.1 200 OK [...] ``` See also: * kubernetes/kubernetes#121120 * kubernetes/kubernetes#121197 * golang/go#63417 (comment) Signed-off-by: Simon Pasquier <spasquie@redhat.com>
prometheus-operator · Oct 19, 2023 · 7da8258 · 7da8258
1 parent 6aefeaf
commit 7da8258
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 10 deletions.
diff --git a/Documentation/operator.md b/Documentation/operator.md
@@ -89,6 +89,8 @@ Usage of ./operator:
     	Cert file to be used for operator web server endpoints. (default "/etc/tls/private/tls.crt")
   -web.client-ca-file string
     	Client CA certificate file to be used for operator web server endpoints. (default "/etc/tls/private/tls-ca.crt")
+  -web.enable-http2
+    	Enable HTTP2 connections.
   -web.enable-tls
     	Activate prometheus operator web server TLS.   This is useful for example when using the rule validation webhook.
   -web.key-file string

diff --git a/cmd/admission-webhook/main.go b/cmd/admission-webhook/main.go
@@ -53,12 +53,22 @@ var (
 	cfg     = config{}
 	flagset = flag.CommandLine
 
+	enableHTTP2        bool
 	serverTLS          bool
 	rawTLSCipherSuites string
 )
 
 func main() {
 	flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8443", "Address on which the admission webhook service listens")
+	// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
+	// standard library and golang.org/x/net are fully fixed.
+	// Right now, it is possible for authenticated and unauthenticated users to
+	// hold open HTTP2 connections and consume huge amounts of memory.
+	// See:
+	// * https://github.com/kubernetes/kubernetes/pull/121120
+	// * https://github.com/kubernetes/kubernetes/issues/121197
+	// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
+	flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.")
 	flagset.BoolVar(&serverTLS, "web.enable-tls", true, "Enable TLS web server")
 
 	flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultCrtFile, "Certificate file to be used for the web server.")
@@ -163,18 +173,23 @@ func newSrv(logger log.Logger, tlsConf *tls.Config) *srv {
 		w.WriteHeader(http.StatusOK)
 	})
 
+	httpServer := http.Server{
+		Handler:           mux,
+		TLSConfig:         tlsConf,
+		ReadHeaderTimeout: 30 * time.Second,
+		ReadTimeout:       30 * time.Second,
+		// use flags on standard logger to align with base logger and get consistent parsed fields form adapter:
+		// use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message)
+		// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
+		ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
+	}
+	if !enableHTTP2 {
+		httpServer.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	}
+
 	return &srv{
 		logger: logger,
-		s: &http.Server{
-			Handler:           mux,
-			TLSConfig:         tlsConf,
-			ReadHeaderTimeout: 30 * time.Second,
-			ReadTimeout:       30 * time.Second,
-			// use flags on standard logger to align with base logger and get consistent parsed fields form adapter:
-			// use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message)
-			// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
-			ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
-		},
+		s:      &httpServer,
 	}
 }
 

diff --git a/cmd/operator/main.go b/cmd/operator/main.go
@@ -161,13 +161,23 @@ var (
 	cfg = operator.DefaultConfig(defaultReloaderCPU, defaultReloaderMemory)
 
 	rawTLSCipherSuites string
+	enableHTTP2        bool
 	serverTLS          bool
 
 	flagset = flag.CommandLine
 )
 
 func init() {
 	flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8080", "Address on which to expose metrics and web interface.")
+	// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
+	// standard library and golang.org/x/net are fully fixed.
+	// Right now, it is possible for authenticated and unauthenticated users to
+	// hold open HTTP2 connections and consume huge amounts of memory.
+	// See:
+	// * https://github.com/kubernetes/kubernetes/pull/121120
+	// * https://github.com/kubernetes/kubernetes/issues/121197
+	// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
+	flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.")
 	flagset.BoolVar(&serverTLS, "web.enable-tls", false, "Activate prometheus operator web server TLS.  "+
 		" This is useful for example when using the rule validation webhook.")
 	flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultOperatorTLSDir+"/tls.crt", "Cert file to be used for operator web server endpoints.")
@@ -525,6 +535,9 @@ func run() int {
 		// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
 		ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
 	}
+	if !enableHTTP2 {
+		srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	}
 	if srv.TLSConfig == nil {
 		wg.Go(serve(srv, l, logger))
 	} else {