Use one HTTP server for healthz, metrics, and profiling.

TLS can still be enabled via flags, but it now applies to all three APIs since there is a single HTTP server per operator process. The -profiling flag is deprecated and does nothing. The profiling APIs are now always enabled, but they refuse to serve clients that do not present a verifiable certificate (see the new flag -client-ca). This effectively disables the profiling APIs when served over HTTP. Co-authored-by: Vu Dinh <vudinh@outlook.com> Co-authored-by: Alexander Greene <agreene@redhat.com> Signed-off-by: Ben Luddy <bluddy@redhat.com>
operator-framework · Jul 14, 2021 · f9381e0 · f9381e0
1 parent 4ffa5c1
commit f9381e0
Show file tree

Hide file tree

Showing 14 changed files with 295 additions and 192 deletions.
diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go
@@ -2,26 +2,23 @@ package main
 
 import (
 	"context"
-	"crypto/tls"
 	"flag"
 	"fmt"
 	"net/http"
 	"os"
 	"time"
 
 	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	utilclock "k8s.io/apimachinery/pkg/util/clock"
 	k8sscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/catalog"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
@@ -67,8 +64,9 @@ var (
 	tlsCertPath = flag.String(
 		"tls-cert", "", "Path to use for certificate key (requires tls-key)")
 
-	profiling = flag.Bool(
-		"profiling", false, "serve profiling data (on port 8080)")
+	profiling = flag.Bool("profiling", false, "deprecated")
+
+	clientCAPath = flag.String("client-ca", "", "path to watch for client ca bundle")
 
 	installPlanTimeout  = flag.Duration("install-plan-retry-timeout", 1*time.Minute, "time since first attempt at which plan execution errors are considered fatal")
 	bundleUnpackTimeout = flag.Duration("bundle-unpack-timeout", 10*time.Minute, "The time limit for bundle unpacking, after which InstallPlan execution is considered to have failed. 0 is considered as having no timeout.")
@@ -106,59 +104,16 @@ func main() {
 		*catalogNamespace = catalogNamespaceEnvVarValue
 	}
 
-	var useTLS bool
-	if *tlsCertPath != "" && *tlsKeyPath == "" || *tlsCertPath == "" && *tlsKeyPath != "" {
-		logger.Warn("both --tls-key and --tls-crt must be provided for TLS to be enabled, falling back to non-https")
-	} else if *tlsCertPath == "" && *tlsKeyPath == "" {
-		logger.Info("TLS keys not set, using non-https for metrics")
-	} else {
-		logger.Info("TLS keys set, using https for metrics")
-		useTLS = true
-	}
-
-	// Serve a health check.
-	healthMux := http.NewServeMux()
-	healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	})
-
-	// Serve profiling if enabled
-	if *profiling {
-		logger.Infof("profiling enabled")
-		profile.RegisterHandlers(healthMux)
+	listenAndServe, err := server.GetListenAndServeFunc(logger, tlsCertPath, tlsKeyPath, clientCAPath)
+	if err != nil {
+		logger.Fatal("Error setting up health/metric/pprof service: %v", err)
 	}
 
-	go http.ListenAndServe(":8080", healthMux)
-
-	metricsMux := http.NewServeMux()
-	metricsMux.Handle("/metrics", promhttp.Handler())
-	if useTLS {
-		tlsGetCertFn, err := filemonitor.OLMGetCertRotationFn(logger, *tlsCertPath, *tlsKeyPath)
-		if err != nil {
-			logger.Errorf("Certificate monitoring for metrics (https) failed: %v", err)
+	go func() {
+		if err := listenAndServe(); err != nil && err != http.ErrServerClosed {
+			logger.Error(err)
 		}
-
-		go func() {
-			httpsServer := &http.Server{
-				Addr:    ":8081",
-				Handler: metricsMux,
-				TLSConfig: &tls.Config{
-					GetCertificate: tlsGetCertFn,
-				},
-			}
-			err := httpsServer.ListenAndServeTLS("", "")
-			if err != nil {
-				logger.Errorf("Metrics (https) serving failed: %v", err)
-			}
-		}()
-	} else {
-		go func() {
-			err := http.ListenAndServe(":8081", metricsMux)
-			if err != nil {
-				logger.Errorf("Metrics (http) serving failed: %v", err)
-			}
-		}()
-	}
+	}()
 
 	// create a config client for operator status
 	config, err := clientcmd.BuildConfigFromFlags("", *kubeConfigPath)

diff --git a/cmd/olm/main.go b/cmd/olm/main.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/tls"
 	"flag"
 	"fmt"
 	"net/http"
@@ -12,7 +11,6 @@ import (
 
 	configclientset "github.com/openshift/client-go/config/clientset/versioned"
 	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
@@ -23,11 +21,10 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/openshift"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/feature"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
@@ -66,8 +63,9 @@ var (
 	tlsCertPath = pflag.String(
 		"tls-cert", "", "Path to use for certificate key (requires tls-key)")
 
-	profiling = pflag.Bool(
-		"profiling", false, "serve profiling data (on port 8080)")
+	profiling = pflag.Bool("profiling", false, "deprecated")
+
+	clientCAPath = pflag.String("client-ca", "", "path to watch for client ca bundle")
 
 	namespace = pflag.String(
 		"namespace", "", "namespace where cleanup runs")
@@ -120,65 +118,17 @@ func main() {
 	}
 	logger.Infof("log level %s", logger.Level)
 
-	var useTLS bool
-	if *tlsCertPath != "" && *tlsKeyPath == "" || *tlsCertPath == "" && *tlsKeyPath != "" {
-		logger.Warn("both --tls-key and --tls-crt must be provided for TLS to be enabled, falling back to non-https")
-	} else if *tlsCertPath == "" && *tlsKeyPath == "" {
-		logger.Info("TLS keys not set, using non-https for metrics")
-	} else {
-		logger.Info("TLS keys set, using https for metrics")
-		useTLS = true
-	}
-
-	// Serve a health check.
-	healthMux := http.NewServeMux()
-	healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	})
-
-	// Serve profiling if enabled
-	if *profiling {
-		logger.Infof("profiling enabled")
-		profile.RegisterHandlers(healthMux)
+	listenAndServe, err := server.GetListenAndServeFunc(logger, tlsCertPath, tlsKeyPath, clientCAPath)
+	if err != nil {
+		logger.Fatal("Error setting up health/metric/pprof service: %v", err)
 	}
 
 	go func() {
-		err := http.ListenAndServe(":8080", healthMux)
-		if err != nil {
-			logger.Errorf("Health serving failed: %v", err)
+		if err := listenAndServe(); err != nil && err != http.ErrServerClosed {
+			logger.Error(err)
 		}
 	}()
 
-	metricsMux := http.NewServeMux()
-	metricsMux.Handle("/metrics", promhttp.Handler())
-	if useTLS {
-		tlsGetCertFn, err := filemonitor.OLMGetCertRotationFn(logger, *tlsCertPath, *tlsKeyPath)
-		if err != nil {
-			logger.Errorf("Certificate monitoring for metrics (https) failed: %v", err)
-		}
-
-		go func() {
-			httpsServer := &http.Server{
-				Addr:    ":8081",
-				Handler: metricsMux,
-				TLSConfig: &tls.Config{
-					GetCertificate: tlsGetCertFn,
-				},
-			}
-			err := httpsServer.ListenAndServeTLS("", "")
-			if err != nil {
-				logger.Errorf("Metrics (https) serving failed: %v", err)
-			}
-		}()
-	} else {
-		go func() {
-			err := http.ListenAndServe(":8081", metricsMux)
-			if err != nil {
-				logger.Errorf("Metrics (http) serving failed: %v", err)
-			}
-		}()
-	}
-
 	mgr, err := Manager(ctx, *debug)
 	if err != nil {
 		logger.WithError(err).Fatalf("error configuring controller manager")

diff --git a/deploy/chart/templates/0000_50_olm_02-services.yaml b/deploy/chart/templates/0000_50_olm_02-services.yaml
@@ -12,9 +12,9 @@ spec:
   type: ClusterIP
   ports:
   - name: https-metrics
-    port: 8081
+    port: {{ .Values.olm.service.externalPort }}
     protocol: TCP
-    targetPort: metrics
+    targetPort: {{ .Values.olm.service.internalPort }}
   selector:
     app: olm-operator
 ---
@@ -31,9 +31,9 @@ spec:
   type: ClusterIP
   ports:
   - name: https-metrics
-    port: 8081
+    port: {{ .Values.catalog.service.externalPort }}
     protocol: TCP
-    targetPort: metrics
+    targetPort: {{ .Values.catalog.service.internalPort }}
   selector:
     app: catalog-operator
 {{ end }}
diff --git a/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml b/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml
@@ -18,8 +18,34 @@ spec:
         app: olm-operator
     spec:
       serviceAccountName: olm-operator-serviceaccount
+      {{- if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}
+      volumes: 
+      {{- end }}
+      {{- if .Values.olm.tlsSecret }}
+      - name: srv-cert
+        secret:
+          secretName: {{ .Values.olm.tlsSecret }}
+      {{- end }}
+      {{- if .Values.olm.clientCASecret }}
+      - name: profile-collector-cert
+        secret:
+          secretName: {{ .Values.olm.clientCASecret }}
+      {{- end }}
       containers:
         - name: olm-operator
+          {{- if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}
+          volumeMounts:
+          {{- end }}
+          {{- if .Values.olm.tlsSecret }}
+          - name: srv-cert
+            mountPath: "/srv-cert"
+            readOnly: true
+          {{- end }}
+          {{- if .Values.olm.clientCASecret }}
+          - name: profile-collector-cert
+            mountPath: "/profile-collector-cert"
+            readOnly: true
+          {{- end }}
           command:
           - /bin/olm
           args:
@@ -43,29 +69,30 @@ spec:
           - --writePackageServerStatusName
           - {{ .Values.writePackageServerStatusName }}
           {{- end }}
-          {{- if .Values.olm.tlsCertPath }}
+         {{- if .Values.olm.tlsSecret }}
           - --tls-cert
-          - {{ .Values.olm.tlsCertPath }}
-          {{- end }}
-          {{- if .Values.olm.tlsKeyPath }}
+          - /srv-cert/tls.crt
           - --tls-key
-          - {{ .Values.olm.tlsKeyPath }}
+          - /srv-cert/tls.key
+          {{- end }}
+          {{- if .Values.olm.clientCASecret }}
+          - --client-ca
+          - /profile-collector-cert/tls.crt
           {{- end }}
           image: {{ .Values.olm.image.ref }}
           imagePullPolicy: {{ .Values.olm.image.pullPolicy }}
           ports:
             - containerPort: {{ .Values.olm.service.internalPort }}
-            - containerPort: 8081
-              name: metrics
-              protocol: TCP
           livenessProbe:
             httpGet:
               path: /healthz
               port: {{ .Values.olm.service.internalPort }}
+              scheme: {{ if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}HTTPS{{ else }}HTTP{{end}}
           readinessProbe:
             httpGet:
               path: /healthz
               port: {{ .Values.olm.service.internalPort }}
+              scheme: {{ if or .Values.olm.tlsSecret .Values.olm.clientCASecret }}HTTPS{{ else }}HTTP{{end}}
           terminationMessagePolicy: FallbackToLogsOnError
           env:
           - name: OPERATOR_NAMESPACE

diff --git a/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml b/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml
@@ -18,8 +18,34 @@ spec:
         app: catalog-operator
     spec:
       serviceAccountName: olm-operator-serviceaccount
+      {{- if or .Values.catalog.tlsSecret .Values.catalog.clientCASecret }}
+      volumes: 
+      {{- end }}
+      {{- if .Values.catalog.tlsSecret }}
+      - name: srv-cert
+        secret:
+          secretName: {{ .Values.catalog.tlsSecret }}
+      {{- end }}
+      {{- if .Values.catalog.clientCASecret }}
+      - name: profile-collector-cert
+        secret:
+          secretName: {{ .Values.catalog.clientCASecret }}
+      {{- end }}
       containers:
         - name: catalog-operator
+          {{- if or .Values.catalog.tlsSecret .Values.catalog.clientCASecret }}
+          volumeMounts:
+          {{- end }}
+          {{- if .Values.catalog.tlsSecret }}
+          - name: srv-cert
+            mountPath: "/srv-cert"
+            readOnly: true
+          {{- end }}
+          {{- if .Values.catalog.clientCASecret }}
+          - name: profile-collector-cert
+            mountPath: "/profile-collector-cert"
+            readOnly: true
+          {{- end }}
           command:
           - /bin/catalog
           args:
@@ -37,29 +63,30 @@ spec:
           - -writeStatusName
           - {{ .Values.writeStatusNameCatalog }}
           {{- end }}
-          {{- if .Values.olm.tlsCertPath }}
-          - -tls-cert
-          - {{ .Values.olm.tlsCertPath }}
+          {{- if .Values.catalog.tlsSecret }}
+          - --tls-cert
+          - /srv-cert/tls.crt
+          - --tls-key
+          - /srv-cert/tls.key
           {{- end }}
-          {{- if .Values.olm.tlsKeyPath }}
-          - -tls-key
-          - {{ .Values.olm.tlsKeyPath }}
+          {{- if .Values.catalog.clientCASecret }}
+          - --client-ca
+          - /profile-collector-cert/tls.crt
           {{- end }}
           image: {{ .Values.catalog.image.ref }}
           imagePullPolicy: {{ .Values.catalog.image.pullPolicy }}
           ports:
             - containerPort: {{ .Values.catalog.service.internalPort }}
-            - containerPort: 8081
-              name: metrics
-              protocol: TCP
           livenessProbe:
             httpGet:
               path: /healthz
               port: {{ .Values.catalog.service.internalPort }}
+              scheme: {{ if and .Values.catalog.tlsKeyPath .Values.catalog.tlsCertPath }}HTTPS{{ else }}HTTP{{end}}
           readinessProbe:
             httpGet:
               path: /healthz
               port: {{ .Values.catalog.service.internalPort }}
+              scheme: {{ if and .Values.catalog.tlsKeyPath .Values.catalog.tlsCertPath }}HTTPS{{ else }}HTTP{{end}}
           terminationMessagePolicy: FallbackToLogsOnError
           {{- if .Values.catalog.resources }}
           resources: