[release-4.4] Bug 1813065: enable prometheus metrics over https

cmd/manager/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/operator-framework/operator-marketplace/pkg/metrics"
 	"net/http"
 	"os"
 	"runtime"
@@ -45,7 +46,11 @@ const (
 	updateNotificationSendWait = time.Duration(10) * time.Minute
 )
 
-var version = flag.Bool("version", false, "displays marketplace source commit info.")
+var (
+	version     = flag.Bool("version", false, "displays marketplace source commit info.")
+	tlsKeyPath  = flag.String("tls-key", "", "Path to use for private key (requires tls-cert)")
+	tlsCertPath = flag.String("tls-cert", "", "Path to use for certificate (requires tls-key)")
+)
 
 func printVersion() {
 	log.Printf("Go Version: %s", runtime.Version())
@@ -72,6 +77,22 @@ func main() {
 		os.Exit(0)
 	}
 
+	// set TLS to serve metrics over a secure channel if cert is provided
+	// cert is provided by default by the marketplace-trusted-ca volume mounted as part of the marketplace-operator deployment
+	var useTLS bool
+	if *tlsCertPath != "" && *tlsKeyPath == "" || *tlsCertPath == "" && *tlsKeyPath != "" {
+		log.Warn("both --tls-key and --tls-crt must be provided for TLS to be enabled, falling back to non-https")
+	} else if *tlsCertPath == "" && *tlsKeyPath == "" {
+		log.Info("TLS keys not set, using non-https for metrics")
+	} else {
+		log.Info("TLS keys set, using https for metrics")
+		useTLS = true
+	}
+	err := metrics.ServePrometheus(useTLS, *tlsCertPath, *tlsKeyPath)
+	if err != nil {
+		log.Fatalf("failed to serve prometheus metrics: TLS enabled %d: %s", useTLS, err)
+	}
+
 	namespace, err := k8sutil.GetWatchNamespace()
 	if err != nil {
 		log.Fatalf("failed to get watch namespace: %v", err)

manifests/08_service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: marketplace-operator-metrics
   namespace: openshift-marketplace
+  annotations:
+    service.alpha.openshift.io/serving-cert-secret-name: marketplace-operator-metrics
   labels:
     name: marketplace-operator
 spec:
@@ -13,3 +15,7 @@ spec:
     port: 8383
     protocol: TCP
     targetPort: 8383
+  - name: https-metrics
+    port: 8081
+    protocol: TCP
+    targetPort: 8081

manifests/09_operator.yaml

@@ -45,6 +45,10 @@ spec:
           - -registryServerImage=quay.io/openshift/origin-operator-registry
           - -defaultsDir=/defaults
           - -clusterOperatorName=marketplace
+          - -tls-cert
+          - /var/run/secrets/serving-cert/tls.crt
+          - -tls-key
+          - /var/run/secrets/serving-cert/tls.key
           imagePullPolicy: IfNotPresent
           livenessProbe:
             httpGet:
@@ -75,6 +79,8 @@ spec:
           volumeMounts:
             - name: marketplace-trusted-ca
               mountPath: /etc/pki/ca-trust/extracted/pem/
+            - name: marketplace-operator-metrics
+              mountPath: /var/run/secrets/serving-cert
       volumes:
         - name: marketplace-trusted-ca
           configMap:
@@ -83,3 +89,6 @@ spec:
             items:
             - key: ca-bundle.crt
               path: tls-ca-bundle.pem
+        - name: marketplace-operator-metrics
+          secret:
+            secretName: marketplace-operator-metrics

manifests/11_service_monitor.yaml

@@ -7,7 +7,18 @@ metadata:
     name: marketplace-operator
 spec:
   endpoints:
-  - port: metrics
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 30s
+      metricRelabelings:
+        - action: drop
+          regex: etcd_(debugging|disk|request|server).*
+          sourceLabels:
+            - __name__
+      port: https-metrics
+      scheme: https
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: marketplace-operator-metrics.openshift-marketplace.svc
   namespaceSelector:
     matchNames:
     - openshift-marketplace

pkg/certificateauthority/caconfigmap.go

@@ -11,14 +11,14 @@ const (
 	// TrustedCaConfigMapName is the name of the Marketplace ConfigMap that stores Certificate Authority bundle.
 	TrustedCaConfigMapName = "marketplace-trusted-ca"
 
-	// trustedCaMountPath is the path to the directory where the Certificate Authority volume should be mounted.
-	trustedCaMountPath = "/etc/pki/ca-trust/extracted/pem/"
+	// TrustedCaMountPath is the path to the directory where the Certificate Authority volume should be mounted.
+	TrustedCaMountPath = "/etc/pki/ca-trust/extracted/pem/"
 
-	// caBundleKey is the key in the ConfigMap that stores Certificate Authoritie bundle.
-	caBundleKey = "ca-bundle.crt"
+	// CABundleKey is the key in the ConfigMap that stores Certificate Authoritie bundle.
+	CABundleKey = "ca-bundle.crt"
 
-	// caBundlePath is the path where we will mount the Certificate Authorities bundle.
-	caBundlePath = "tls-ca-bundle.pem"
+	// CABundlePath is the path where we will mount the Certificate Authorities bundle.
+	CABundlePath = "tls-ca-bundle.pem"
 )
 
 // MountCaConfigMap adds a Volume and VolumeMount for the Certificate Authority ConfigMap on
@@ -35,8 +35,8 @@ func MountCaConfigMap(template *corev1.PodTemplateSpec) {
 					},
 					Items: []corev1.KeyToPath{
 						corev1.KeyToPath{
-							Key:  caBundleKey,
-							Path: caBundlePath,
+							Key:  CABundleKey,
+							Path: CABundlePath,
 						},
 					},
 				},
@@ -48,13 +48,13 @@ func MountCaConfigMap(template *corev1.PodTemplateSpec) {
 	template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
 		corev1.VolumeMount{
 			Name:      TrustedCaConfigMapName,
-			MountPath: trustedCaMountPath,
+			MountPath: TrustedCaMountPath,
 		},
 	}
 }
 
 // getCaOnDisk returns the contents of the Certificate Authority bundle on disk as a byte
 // array or returns the error encountered when attempting to do so.
 func getCaOnDisk() ([]byte, error) {
-	return ioutil.ReadFile(filepath.Join(trustedCaMountPath, caBundlePath))
+	return ioutil.ReadFile(filepath.Join(TrustedCaMountPath, CABundlePath))
 }

pkg/certificateauthority/handler.go

@@ -39,7 +39,7 @@ func (h *configmapHandler) Handle(ctx context.Context, in *corev1.ConfigMap) err
 	})
 
 	// Retrieve the Certificate Authority bundle from the ConfigMap.
-	caBundle := in.Data[caBundleKey]
+	caBundle := in.Data[CABundleKey]
 
 	// Retrieve the Certificate Authority bundle from Disk.
 	// If an error is returned and is not related to a nonexistant file, return an error.

pkg/filemonitor/cert_updater.go

@@ -0,0 +1,93 @@
+package filemonitor
+
+// filemonitor package is copied from the OLM repository - see below package for original
+// https://github.com/operator-framework/operator-lifecycle-manager/tree/master/pkg/lib/filemonitor
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/sirupsen/logrus"
+)
+
+type keystore struct {
+	mutex      sync.RWMutex
+	cert       *tls.Certificate
+	tlsCrtPath string
+	tlsKeyPath string
+}
+
+type getCertFn = func(*tls.ClientHelloInfo) (*tls.Certificate, error)
+
+// NewKeystore returns a store for storing the certificate data and the ability to retrieve it safely
+func NewKeystore(tlsCrt, tlsKey string) *keystore {
+	cert, err := tls.LoadX509KeyPair(tlsCrt, tlsKey)
+	if err != nil {
+		panic(err)
+	}
+	return &keystore{
+		mutex:      sync.RWMutex{},
+		cert:       &cert,
+		tlsCrtPath: tlsCrt,
+		tlsKeyPath: tlsKey,
+	}
+}
+
+// HandleFilesystemUpdate is intended to be used as the OnUpdateFn for a watcher
+// and expects the certificate files to be in the same directory.
+func (k *keystore) HandleFilesystemUpdate(logger *logrus.Logger, event fsnotify.Event) {
+	switch op := event.Op; op {
+	case fsnotify.Create:
+		logger.Debugf("got fs event for %v", event.Name)
+
+		if err := k.storeCertificate(k.tlsCrtPath, k.tlsKeyPath); err != nil {
+			// this can happen if both certificates aren't updated at the same
+			// time, but it's okay as replacement only occurs with a valid key pair
+			logger.Debugf("certificates not in sync: %v", err)
+		} else {
+			info, err := x509.ParseCertificate(k.cert.Certificate[0])
+			if err != nil {
+				logger.Debugf("certificates refreshed, but parsing returned error: %v", err)
+			} else {
+				logger.Debugf("certificates refreshed: Subject=%v NotBefore=%v NotAfter=%v", info.Subject, info.NotBefore, info.NotAfter)
+			}
+		}
+	}
+}
+
+func (k *keystore) storeCertificate(tlsCrt, tlsKey string) error {
+	cert, err := tls.LoadX509KeyPair(tlsCrt, tlsKey)
+	if err == nil {
+		k.mutex.Lock()
+		defer k.mutex.Unlock()
+		k.cert = &cert
+	}
+	return err
+}
+
+func (k *keystore) GetCertificate(h *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	k.mutex.RLock()
+	defer k.mutex.RUnlock()
+	return k.cert, nil
+}
+
+// OLMGetCertRotationFn is a convenience function for OLM use only, but serves as an example for monitoring file system events
+func OLMGetCertRotationFn(logger *logrus.Logger, tlsCertPath, tlsKeyPath string) (getCertFn, error) {
+	if filepath.Dir(tlsCertPath) != filepath.Dir(tlsKeyPath) {
+		return nil, fmt.Errorf("certificates expected to be in same directory %v vs %v", tlsCertPath, tlsKeyPath)
+	}
+
+	keystore := NewKeystore(tlsCertPath, tlsKeyPath)
+	watcher, err := NewWatch(logger, []string{filepath.Dir(tlsCertPath)}, keystore.HandleFilesystemUpdate)
+	if err != nil {
+		return nil, err
+	}
+	watcher.Run(context.Background())
+
+	return keystore.GetCertificate, nil
+}