metrics: Do not rely on global HTTP server

This moves away from the global Golang HTTP server for serving the
metrics for both Hubble and Cilium. Instead, a local instance is
created, which allows the two metrics server to run concurrently.

Fixes: #11066

Signed-off-by: Sebastian Wicki <sebastian@isovalent.com>

pkg/hubble/metrics/metrics.go

@@ -56,8 +56,13 @@ func Init(address string, enabled api.Map) (<-chan error, error) {
 	errChan := make(chan error, 1)
 
 	go func() {
-		http.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
-		errChan <- http.ListenAndServe(address, nil)
+		mux := http.NewServeMux()
+		mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+		srv := http.Server{
+			Addr:    address,
+			Handler: mux,
+		}
+		errChan <- srv.ListenAndServe()
 	}()
 
 	return errChan, nil

pkg/metrics/metrics.go

@@ -1123,8 +1123,13 @@ func Enable(addr string) <-chan error {
 	go func() {
 		// The Handler function provides a default handler to expose metrics
 		// via an HTTP server. "/metrics" is the usual endpoint for that.
-		http.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
-		errs <- http.ListenAndServe(addr, nil)
+		mux := http.NewServeMux()
+		mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+		srv := http.Server{
+			Addr:    addr,
+			Handler: mux,
+		}
+		errs <- srv.ListenAndServe()
 	}()
 
 	return errs