Skip to content

Commit

Permalink
fix: disable HTTP2 connections by default
Browse files Browse the repository at this point in the history 
This change mitigates CVE-2023-44487 by disabling HTTP2 by default and
forcing HTTP/1.1 until the Go standard library and golang.org/x/net are
fully fixed. Right now, it is possible for authenticated and
unauthenticated users to hold open HTTP2 connections and consume huge
amounts of memory.

It is possible to revert back the change by using the
`--web.enable-http2` argument.

Before this change:

```
curl -kv https://localhost:8443/metrics
*   Trying 127.0.0.1:8443...
* Connected to localhost (127.0.0.1) port 8443 (#0)
* ALPN: offers h2,http/1.1
[...]
* ALPN: server accepted h2
[...]
* using HTTP/2
* h2h3 [:method: GET]
* h2h3 [:path: /metrics]
* h2h3 [:scheme: https]
* h2h3 [:authority: localhost:8443]
* h2h3 [user-agent: curl/8.0.1]
* h2h3 [accept: */*]
* Using Stream ID: 1 (easy handle 0x5594d4614b10)
[...]
> GET /metrics HTTP/2
[...]
```

After this change:

```
curl -kv https://localhost:8443/metrics
*   Trying 127.0.0.1:8443...
* Connected to localhost (127.0.0.1) port 8443 (#0)
* ALPN: offers h2,http/1.1
[...]
* ALPN: server accepted http/1.1
[...]
* using HTTP/1.1
> GET /metrics HTTP/1.1
> Host: localhost:8443
> User-Agent: curl/8.0.1
> Accept: */*
[...]
< HTTP/1.1 200 OK
[...]
```

See also:
* kubernetes/kubernetes#121120
* kubernetes/kubernetes#121197
* golang/go#63417 (comment)

Signed-off-by: Simon Pasquier <spasquie@redhat.com>
  • Loading branch information
@simonpasquier
simonpasquier committed Oct 19, 2023
1 parent 6aefeaf commit a62e814
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 12 deletions.
2 changes: 2 additions & 0 deletions Documentation/operator.md
View file
Expand Up @@ -89,6 +89,8 @@ Usage of ./operator:
Cert file to be used for operator web server endpoints. (default "/etc/tls/private/tls.crt")
-web.client-ca-file string
Client CA certificate file to be used for operator web server endpoints. (default "/etc/tls/private/tls-ca.crt")
-web.enable-http2
Enable HTTP2 connections.
-web.enable-tls
Activate prometheus operator web server TLS. This is useful for example when using the rule validation webhook.
-web.key-file string
Expand Down
37 changes: 26 additions & 11 deletions cmd/admission-webhook/main.go
View file
Expand Up @@ -53,12 +53,22 @@ var (
cfg = config{}
flagset = flag.CommandLine

enableHTTP2 bool
serverTLS bool
rawTLSCipherSuites string
)

func main() {
flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8443", "Address on which the admission webhook service listens")
// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
// standard library and golang.org/x/net are fully fixed.
// Right now, it is possible for authenticated and unauthenticated users to
// hold open HTTP2 connections and consume huge amounts of memory.
// See:
// * https://github.com/kubernetes/kubernetes/pull/121120
// * https://github.com/kubernetes/kubernetes/issues/121197
// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.")
flagset.BoolVar(&serverTLS, "web.enable-tls", true, "Enable TLS web server")

flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultCrtFile, "Certificate file to be used for the web server.")
Expand Down Expand Up @@ -127,7 +137,7 @@ func (s *srv) run(listener net.Listener) error {
log := log.With(s.logger, "address", listener.Addr().String())

if s.s.TLSConfig != nil {
level.Info(log).Log("msg", "Starting TLS enabled server")
level.Info(log).Log("msg", "Starting TLS enabled server", "http2", enableHTTP2)
if err := s.s.ServeTLS(listener, "", ""); err != http.ErrServerClosed {
return err
}
Expand Down Expand Up @@ -163,18 +173,23 @@ func newSrv(logger log.Logger, tlsConf *tls.Config) *srv {
w.WriteHeader(http.StatusOK)
})

httpServer := http.Server{
Handler: mux,
TLSConfig: tlsConf,
ReadHeaderTimeout: 30 * time.Second,
ReadTimeout: 30 * time.Second,
// use flags on standard logger to align with base logger and get consistent parsed fields form adapter:
// use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message)
// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
}
if !enableHTTP2 {
httpServer.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
}

return &srv{
logger: logger,
s: &http.Server{
Handler: mux,
TLSConfig: tlsConf,
ReadHeaderTimeout: 30 * time.Second,
ReadTimeout: 30 * time.Second,
// use flags on standard logger to align with base logger and get consistent parsed fields form adapter:
// use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message)
// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
},
s: &httpServer,
}
}

Expand Down
15 changes: 14 additions & 1 deletion cmd/operator/main.go
View file
Expand Up @@ -144,7 +144,7 @@ func serve(srv *http.Server, listener net.Listener, logger log.Logger) func() er

func serveTLS(srv *http.Server, listener net.Listener, logger log.Logger) func() error {
return func() error {
level.Info(logger).Log("msg", "Starting secure server on "+listener.Addr().String())
level.Info(logger).Log("msg", "Starting secure server on "+listener.Addr().String(), "http2", enableHTTP2)
if err := srv.ServeTLS(listener, "", ""); err != http.ErrServerClosed {
return err
}
Expand All @@ -161,13 +161,23 @@ var (
cfg = operator.DefaultConfig(defaultReloaderCPU, defaultReloaderMemory)

rawTLSCipherSuites string
enableHTTP2 bool
serverTLS bool

flagset = flag.CommandLine
)

func init() {
flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8080", "Address on which to expose metrics and web interface.")
// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
// standard library and golang.org/x/net are fully fixed.
// Right now, it is possible for authenticated and unauthenticated users to
// hold open HTTP2 connections and consume huge amounts of memory.
// See:
// * https://github.com/kubernetes/kubernetes/pull/121120
// * https://github.com/kubernetes/kubernetes/issues/121197
// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.")
flagset.BoolVar(&serverTLS, "web.enable-tls", false, "Activate prometheus operator web server TLS. "+
" This is useful for example when using the rule validation webhook.")
flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultOperatorTLSDir+"/tls.crt", "Cert file to be used for operator web server endpoints.")
Expand Down Expand Up @@ -525,6 +535,9 @@ func run() int {
// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
}
if !enableHTTP2 {
srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
}
if srv.TLSConfig == nil {
wg.Go(serve(srv, l, logger))
} else {
Expand Down

0 comments on commit a62e814

Please sign in to comment.