From 65b83b6efcb5f15549b70058e7e761a199b39147 Mon Sep 17 00:00:00 2001 From: Anik Bhattacharjee Date: Tue, 16 Sep 2025 10:16:26 -0400 Subject: [PATCH 1/5] WIP: Authenticated metrics endpoints using controller-runtime auth filters --- ...50_olm_02-olm-operator.serviceaccount.yaml | 12 ++++ ...50_olm_02-olm-operator.serviceaccount.yaml | 12 ++++ .../cmd/catalog/main.go | 12 ++-- .../cmd/olm/main.go | 19 +++--- ...50_olm_02-olm-operator.serviceaccount.yaml | 12 ++++ .../pkg/lib/server/server.go | 63 ++++++++++++------- 6 files changed, 96 insertions(+), 34 deletions(-) diff --git a/manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml b/manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml index 8e9ea5f101..febe778850 100644 --- a/manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml +++ b/manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml @@ -24,6 +24,18 @@ rules: verbs: ["watch", "list", "get", "create", "update", "patch", "delete", "deletecollection", "escalate", "bind"] - nonResourceURLs: ["*"] verbs: ["*"] + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - security.openshift.io resources: diff --git a/microshift-manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml b/microshift-manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml index 8e9ea5f101..febe778850 100644 --- a/microshift-manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml +++ b/microshift-manifests/0000_50_olm_02-olm-operator.serviceaccount.yaml @@ -24,6 +24,18 @@ rules: verbs: ["watch", "list", "get", "create", "update", "patch", "delete", "deletecollection", "escalate", "bind"] - nonResourceURLs: ["*"] verbs: ["*"] + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - security.openshift.io resources: diff --git a/staging/operator-lifecycle-manager/cmd/catalog/main.go b/staging/operator-lifecycle-manager/cmd/catalog/main.go index b82f1689cb..afe3f6781f 100644 --- a/staging/operator-lifecycle-manager/cmd/catalog/main.go +++ b/staging/operator-lifecycle-manager/cmd/catalog/main.go @@ -57,9 +57,15 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error { o.catalogNamespace = catalogNamespaceEnvVarValue } + config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) + if err != nil { + return fmt.Errorf("error configuring client: %s", err.Error()) + } + listenAndServe, err := server.GetListenAndServeFunc( server.WithLogger(logger), server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath), + server.WithKubeConfig(config), server.WithDebug(o.debug), ) if err != nil { @@ -71,12 +77,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error { logger.Error(err) } }() - - // create a config client for operator status - config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) - if err != nil { - return fmt.Errorf("error configuring client: %s", err.Error()) - } configClient, err := configv1client.NewForConfig(config) if err != nil { return fmt.Errorf("error configuring client: %s", err.Error()) diff --git a/staging/operator-lifecycle-manager/cmd/olm/main.go b/staging/operator-lifecycle-manager/cmd/olm/main.go index ff6e7354e6..a5ea596910 100644 --- a/staging/operator-lifecycle-manager/cmd/olm/main.go +++ b/staging/operator-lifecycle-manager/cmd/olm/main.go @@ -123,7 +123,18 @@ func main() { } logger.Infof("log level %s", logger.Level) - listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug)) + mgr, err := Manager(ctx, *debug) + if err != nil { + logger.WithError(err).Fatal("error configuring controller manager") + } + config := mgr.GetConfig() + + listenAndServe, err := server.GetListenAndServeFunc( + server.WithLogger(logger), + server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), + server.WithKubeConfig(config), + server.WithDebug(*debug), + ) if err != nil { logger.Fatalf("Error setting up health/metric/pprof service: %v", err) } @@ -134,12 +145,6 @@ func main() { } }() - mgr, err := Manager(ctx, *debug) - if err != nil { - logger.WithError(err).Fatal("error configuring controller manager") - } - config := mgr.GetConfig() - // create a config that validates we're creating objects with labels validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme()) diff --git a/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olm-operator.serviceaccount.yaml b/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olm-operator.serviceaccount.yaml index fceffd024c..ee46632971 100644 --- a/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olm-operator.serviceaccount.yaml +++ b/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olm-operator.serviceaccount.yaml @@ -8,6 +8,18 @@ rules: verbs: ["watch", "list", "get", "create", "update", "patch", "delete", "deletecollection", "escalate", "bind"] - nonResourceURLs: ["*"] verbs: ["*"] +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create --- kind: ServiceAccount apiVersion: v1 diff --git a/staging/operator-lifecycle-manager/pkg/lib/server/server.go b/staging/operator-lifecycle-manager/pkg/lib/server/server.go index 3d79a192e0..a9101a221b 100644 --- a/staging/operator-lifecycle-manager/pkg/lib/server/server.go +++ b/staging/operator-lifecycle-manager/pkg/lib/server/server.go @@ -6,11 +6,16 @@ import ( "fmt" "net/http" "path/filepath" + "time" + "github.com/go-logr/logr" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + "sigs.k8s.io/controller-runtime/pkg/log" ) // Option applies a configuration option to the given config. @@ -43,11 +48,18 @@ func WithDebug(debug bool) Option { } } +func WithKubeConfig(config *rest.Config) Option { + return func(sc *serverConfig) { + sc.kubeConfig = config + } +} + type serverConfig struct { logger *logrus.Logger tlsCertPath *string tlsKeyPath *string clientCAPath *string + kubeConfig *rest.Config debug bool } @@ -62,6 +74,7 @@ func defaultServerConfig() serverConfig { tlsCertPath: nil, tlsKeyPath: nil, clientCAPath: nil, + kubeConfig: nil, logger: nil, debug: false, } @@ -89,13 +102,40 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { return nil, fmt.Errorf("both --tls-key and --tls-crt must be provided for TLS to be enabled") } + // Create base HTTP mux with unprotected endpoints mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) profile.RegisterHandlers(mux, profile.WithTLS(tlsEnabled || !sc.debug)) + // Set up authenticated metrics endpoint if kubeConfig is provided + if sc.kubeConfig != nil && tlsEnabled { + // Create authentication filter using controller-runtime + filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{ + Timeout: 30 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("failed to create authentication filter: %w", err) + } + + // Create authenticated metrics handler + logger := log.FromContext(context.Background()) + authenticatedMetricsHandler, err := filter(logger, promhttp.Handler()) + if err != nil { + return nil, fmt.Errorf("failed to wrap metrics handler with authentication: %w", err) + } + + mux.Handle("/metrics", authenticatedMetricsHandler) + sc.logger.Info("Metrics endpoint configured with authentication and authorization") + } else { + // Fallback to unprotected metrics (for development/testing) + mux.Handle("/metrics", promhttp.Handler()) + if sc.kubeConfig == nil { + sc.logger.Warn("No Kubernetes config provided - metrics endpoint will be unprotected") + } + } + s := http.Server{ Handler: mux, Addr: sc.getAddress(tlsEnabled), @@ -116,31 +156,12 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { return nil, fmt.Errorf("error creating cert file watcher: %v", err) } csw.Run(context.Background()) - certPoolStore, err := filemonitor.NewCertPoolStore(*sc.clientCAPath) - if err != nil { - return nil, fmt.Errorf("certificate monitoring for client-ca failed: %v", err) - } - cpsw, err := filemonitor.NewWatch(sc.logger, []string{filepath.Dir(*sc.clientCAPath)}, certPoolStore.HandleCABundleUpdate) - if err != nil { - return nil, fmt.Errorf("error creating cert file watcher: %v", err) - } - cpsw.Run(context.Background()) s.TLSConfig = &tls.Config{ GetCertificate: func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { return certStore.GetCertificate(), nil }, - GetConfigForClient: func(_ *tls.ClientHelloInfo) (*tls.Config, error) { - var certs []tls.Certificate - if cert := certStore.GetCertificate(); cert != nil { - certs = append(certs, *cert) - } - return &tls.Config{ - Certificates: certs, - ClientCAs: certPoolStore.GetCertPool(), - ClientAuth: tls.VerifyClientCertIfGiven, - }, nil - }, + NextProtos: []string{"http/1.1"}, // Disable HTTP/2 for security } return func() error { return s.ListenAndServeTLS("", "") From ba7e0e2284681d952c63e0d4c2b1e35d4e91ca58 Mon Sep 17 00:00:00 2001 From: Anik Bhattacharjee Date: Tue, 16 Sep 2025 16:12:23 -0400 Subject: [PATCH 2/5] add debugging logs --- ...operator.deployment.ibm-cloud-managed.yaml | 3 +++ ...000_50_olm_07-olm-operator.deployment.yaml | 3 +++ ...operator.deployment.ibm-cloud-managed.yaml | 9 +++++--- ...50_olm_08-catalog-operator.deployment.yaml | 9 +++++--- ...operator.deployment.ibm-cloud-managed.yaml | 3 +++ ...000_50_olm_07-olm-operator.deployment.yaml | 3 +++ ...operator.deployment.ibm-cloud-managed.yaml | 9 +++++--- ...50_olm_08-catalog-operator.deployment.yaml | 9 +++++--- .../deploy/chart/values.yaml | 2 +- .../pkg/lib/server/server.go | 22 ++++++++++++++++++- 10 files changed, 58 insertions(+), 14 deletions(-) diff --git a/manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml b/manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml index 63cde3a8da..8bf93ac5cd 100644 --- a/manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml +++ b/manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml @@ -59,6 +59,7 @@ spec: args: - --namespace - $(OPERATOR_NAMESPACE) + - --debug - --writeStatusName - operator-lifecycle-manager - --writePackageServerStatusName @@ -94,6 +95,8 @@ spec: fieldPath: metadata.namespace - name: OPERATOR_NAME value: olm-operator + - name: CI + value: "true" - name: RELEASE_VERSION value: "0.0.1-snapshot" resources: diff --git a/manifests/0000_50_olm_07-olm-operator.deployment.yaml b/manifests/0000_50_olm_07-olm-operator.deployment.yaml index 0463328554..f74d2c10e9 100644 --- a/manifests/0000_50_olm_07-olm-operator.deployment.yaml +++ b/manifests/0000_50_olm_07-olm-operator.deployment.yaml @@ -58,6 +58,7 @@ spec: args: - --namespace - $(OPERATOR_NAMESPACE) + - --debug - --writeStatusName - operator-lifecycle-manager - --writePackageServerStatusName @@ -93,6 +94,8 @@ spec: fieldPath: metadata.namespace - name: OPERATOR_NAME value: olm-operator + - name: CI + value: "true" - name: RELEASE_VERSION value: "0.0.1-snapshot" resources: diff --git a/manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml b/manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml index 61c35d18c8..50b6e29ec1 100644 --- a/manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml +++ b/manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml @@ -59,6 +59,7 @@ spec: args: - '--namespace' - openshift-marketplace + - '--debug' - --configmapServerImage=quay.io/operator-framework/configmap-operator-registry:latest - --opmImage=quay.io/operator-framework/configmap-operator-registry:latest - --util-image @@ -73,6 +74,11 @@ spec: - /profile-collector-cert/tls.crt - --set-workload-user-id=false image: quay.io/operator-framework/olm@sha256:de396b540b82219812061d0d753440d5655250c621c753ed1dc67d6154741607 + env: + - name: CI + value: "true" + - name: RELEASE_VERSION + value: "0.0.1-snapshot" imagePullPolicy: IfNotPresent ports: - containerPort: 8443 @@ -92,9 +98,6 @@ spec: requests: cpu: 10m memory: 80Mi - env: - - name: RELEASE_VERSION - value: "0.0.1-snapshot" nodeSelector: kubernetes.io/os: linux tolerations: diff --git a/manifests/0000_50_olm_08-catalog-operator.deployment.yaml b/manifests/0000_50_olm_08-catalog-operator.deployment.yaml index 238b91d9a2..f4bf074dc4 100644 --- a/manifests/0000_50_olm_08-catalog-operator.deployment.yaml +++ b/manifests/0000_50_olm_08-catalog-operator.deployment.yaml @@ -58,6 +58,7 @@ spec: args: - '--namespace' - openshift-marketplace + - '--debug' - --configmapServerImage=quay.io/operator-framework/configmap-operator-registry:latest - --opmImage=quay.io/operator-framework/configmap-operator-registry:latest - --util-image @@ -72,6 +73,11 @@ spec: - /profile-collector-cert/tls.crt - --set-workload-user-id=false image: quay.io/operator-framework/olm@sha256:de396b540b82219812061d0d753440d5655250c621c753ed1dc67d6154741607 + env: + - name: CI + value: "true" + - name: RELEASE_VERSION + value: "0.0.1-snapshot" imagePullPolicy: IfNotPresent ports: - containerPort: 8443 @@ -91,9 +97,6 @@ spec: requests: cpu: 10m memory: 80Mi - env: - - name: RELEASE_VERSION - value: "0.0.1-snapshot" nodeSelector: kubernetes.io/os: linux node-role.kubernetes.io/master: "" diff --git a/microshift-manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml b/microshift-manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml index 63cde3a8da..8bf93ac5cd 100644 --- a/microshift-manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml +++ b/microshift-manifests/0000_50_olm_07-olm-operator.deployment.ibm-cloud-managed.yaml @@ -59,6 +59,7 @@ spec: args: - --namespace - $(OPERATOR_NAMESPACE) + - --debug - --writeStatusName - operator-lifecycle-manager - --writePackageServerStatusName @@ -94,6 +95,8 @@ spec: fieldPath: metadata.namespace - name: OPERATOR_NAME value: olm-operator + - name: CI + value: "true" - name: RELEASE_VERSION value: "0.0.1-snapshot" resources: diff --git a/microshift-manifests/0000_50_olm_07-olm-operator.deployment.yaml b/microshift-manifests/0000_50_olm_07-olm-operator.deployment.yaml index 856795b652..1bebc5fdfa 100644 --- a/microshift-manifests/0000_50_olm_07-olm-operator.deployment.yaml +++ b/microshift-manifests/0000_50_olm_07-olm-operator.deployment.yaml @@ -58,6 +58,7 @@ spec: args: - --namespace - $(OPERATOR_NAMESPACE) + - --debug - --tls-cert - /srv-cert/tls.crt - --tls-key @@ -89,6 +90,8 @@ spec: fieldPath: metadata.namespace - name: OPERATOR_NAME value: olm-operator + - name: CI + value: "true" - name: RELEASE_VERSION value: "0.0.1-snapshot" resources: diff --git a/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml b/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml index 61c35d18c8..50b6e29ec1 100644 --- a/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml +++ b/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.ibm-cloud-managed.yaml @@ -59,6 +59,7 @@ spec: args: - '--namespace' - openshift-marketplace + - '--debug' - --configmapServerImage=quay.io/operator-framework/configmap-operator-registry:latest - --opmImage=quay.io/operator-framework/configmap-operator-registry:latest - --util-image @@ -73,6 +74,11 @@ spec: - /profile-collector-cert/tls.crt - --set-workload-user-id=false image: quay.io/operator-framework/olm@sha256:de396b540b82219812061d0d753440d5655250c621c753ed1dc67d6154741607 + env: + - name: CI + value: "true" + - name: RELEASE_VERSION + value: "0.0.1-snapshot" imagePullPolicy: IfNotPresent ports: - containerPort: 8443 @@ -92,9 +98,6 @@ spec: requests: cpu: 10m memory: 80Mi - env: - - name: RELEASE_VERSION - value: "0.0.1-snapshot" nodeSelector: kubernetes.io/os: linux tolerations: diff --git a/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.yaml b/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.yaml index 238b91d9a2..f4bf074dc4 100644 --- a/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.yaml +++ b/microshift-manifests/0000_50_olm_08-catalog-operator.deployment.yaml @@ -58,6 +58,7 @@ spec: args: - '--namespace' - openshift-marketplace + - '--debug' - --configmapServerImage=quay.io/operator-framework/configmap-operator-registry:latest - --opmImage=quay.io/operator-framework/configmap-operator-registry:latest - --util-image @@ -72,6 +73,11 @@ spec: - /profile-collector-cert/tls.crt - --set-workload-user-id=false image: quay.io/operator-framework/olm@sha256:de396b540b82219812061d0d753440d5655250c621c753ed1dc67d6154741607 + env: + - name: CI + value: "true" + - name: RELEASE_VERSION + value: "0.0.1-snapshot" imagePullPolicy: IfNotPresent ports: - containerPort: 8443 @@ -91,9 +97,6 @@ spec: requests: cpu: 10m memory: 80Mi - env: - - name: RELEASE_VERSION - value: "0.0.1-snapshot" nodeSelector: kubernetes.io/os: linux node-role.kubernetes.io/master: "" diff --git a/staging/operator-lifecycle-manager/deploy/chart/values.yaml b/staging/operator-lifecycle-manager/deploy/chart/values.yaml index 4e4ee726b8..66faf0620d 100644 --- a/staging/operator-lifecycle-manager/deploy/chart/values.yaml +++ b/staging/operator-lifecycle-manager/deploy/chart/values.yaml @@ -17,7 +17,7 @@ operator_namespace_psa: minKubeVersion: 1.11.0 writeStatusName: '""' imagestream: false -debug: false +debug: true installType: upstream catalogGrpcPodPort: 50051 diff --git a/staging/operator-lifecycle-manager/pkg/lib/server/server.go b/staging/operator-lifecycle-manager/pkg/lib/server/server.go index a9101a221b..958f4e1594 100644 --- a/staging/operator-lifecycle-manager/pkg/lib/server/server.go +++ b/staging/operator-lifecycle-manager/pkg/lib/server/server.go @@ -110,29 +110,49 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { profile.RegisterHandlers(mux, profile.WithTLS(tlsEnabled || !sc.debug)) // Set up authenticated metrics endpoint if kubeConfig is provided + sc.logger.Infof("DEBUG: Checking authentication setup - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled) + if sc.kubeConfig != nil && tlsEnabled { + sc.logger.Info("DEBUG: Setting up authenticated metrics endpoint") + // Create authentication filter using controller-runtime + sc.logger.Info("DEBUG: Creating authentication filter with controller-runtime") filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{ Timeout: 30 * time.Second, }) if err != nil { + sc.logger.Errorf("DEBUG: Failed to create authentication filter: %v", err) return nil, fmt.Errorf("failed to create authentication filter: %w", err) } + sc.logger.Info("DEBUG: Authentication filter created successfully") // Create authenticated metrics handler + sc.logger.Info("DEBUG: Wrapping metrics handler with authentication") logger := log.FromContext(context.Background()) authenticatedMetricsHandler, err := filter(logger, promhttp.Handler()) if err != nil { + sc.logger.Errorf("DEBUG: Failed to wrap metrics handler: %v", err) return nil, fmt.Errorf("failed to wrap metrics handler with authentication: %w", err) } + sc.logger.Info("DEBUG: Metrics handler wrapped successfully") - mux.Handle("/metrics", authenticatedMetricsHandler) + // Add debugging wrapper to log authentication attempts + debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s", + r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent")) + authenticatedMetricsHandler.ServeHTTP(w, r) + }) + + mux.Handle("/metrics", debugAuthHandler) sc.logger.Info("Metrics endpoint configured with authentication and authorization") } else { // Fallback to unprotected metrics (for development/testing) + sc.logger.Warnf("DEBUG: Using unprotected metrics - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled) mux.Handle("/metrics", promhttp.Handler()) if sc.kubeConfig == nil { sc.logger.Warn("No Kubernetes config provided - metrics endpoint will be unprotected") + } else if !tlsEnabled { + sc.logger.Warn("TLS not enabled - metrics endpoint will be unprotected") } } From fd398591e33287acc1d3717524f325007dd42503 Mon Sep 17 00:00:00 2001 From: Anik Bhattacharjee Date: Tue, 16 Sep 2025 18:44:30 -0400 Subject: [PATCH 3/5] restore mTLS setup with client cert verification --- .../pkg/lib/server/server.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/staging/operator-lifecycle-manager/pkg/lib/server/server.go b/staging/operator-lifecycle-manager/pkg/lib/server/server.go index 958f4e1594..0586f26572 100644 --- a/staging/operator-lifecycle-manager/pkg/lib/server/server.go +++ b/staging/operator-lifecycle-manager/pkg/lib/server/server.go @@ -177,10 +177,31 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { } csw.Run(context.Background()) + certPoolStore, err := filemonitor.NewCertPoolStore(*sc.clientCAPath) + if err != nil { + return nil, fmt.Errorf("certificate monitoring for client-ca failed: %v", err) + } + cpsw, err := filemonitor.NewWatch(sc.logger, []string{filepath.Dir(*sc.clientCAPath)}, certPoolStore.HandleCABundleUpdate) + if err != nil { + return nil, fmt.Errorf("error creating cert file watcher: %v", err) + } + cpsw.Run(context.Background()) + s.TLSConfig = &tls.Config{ GetCertificate: func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { return certStore.GetCertificate(), nil }, + GetConfigForClient: func(_ *tls.ClientHelloInfo) (*tls.Config, error) { + var certs []tls.Certificate + if cert := certStore.GetCertificate(); cert != nil { + certs = append(certs, *cert) + } + return &tls.Config{ + Certificates: certs, + ClientCAs: certPoolStore.GetCertPool(), + ClientAuth: tls.VerifyClientCertIfGiven, + }, nil + }, NextProtos: []string{"http/1.1"}, // Disable HTTP/2 for security } return func() error { From 5d3d6fbca9a2cbc8a26ecd2c7027937a7be559f7 Mon Sep 17 00:00:00 2001 From: Anik Bhattacharjee Date: Fri, 19 Sep 2025 09:56:05 -0400 Subject: [PATCH 4/5] run go mod vendor --- .../cmd/catalog/main.go | 12 +- .../cmd/olm/main.go | 19 ++- .../pkg/lib/server/server.go | 64 ++++++++- vendor/modules.txt | 1 + .../pkg/metrics/filters/filters.go | 122 ++++++++++++++++++ 5 files changed, 204 insertions(+), 14 deletions(-) create mode 100644 vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/catalog/main.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/catalog/main.go index b82f1689cb..afe3f6781f 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/catalog/main.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/catalog/main.go @@ -57,9 +57,15 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error { o.catalogNamespace = catalogNamespaceEnvVarValue } + config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) + if err != nil { + return fmt.Errorf("error configuring client: %s", err.Error()) + } + listenAndServe, err := server.GetListenAndServeFunc( server.WithLogger(logger), server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath), + server.WithKubeConfig(config), server.WithDebug(o.debug), ) if err != nil { @@ -71,12 +77,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error { logger.Error(err) } }() - - // create a config client for operator status - config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) - if err != nil { - return fmt.Errorf("error configuring client: %s", err.Error()) - } configClient, err := configv1client.NewForConfig(config) if err != nil { return fmt.Errorf("error configuring client: %s", err.Error()) diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go index ff6e7354e6..a5ea596910 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go @@ -123,7 +123,18 @@ func main() { } logger.Infof("log level %s", logger.Level) - listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug)) + mgr, err := Manager(ctx, *debug) + if err != nil { + logger.WithError(err).Fatal("error configuring controller manager") + } + config := mgr.GetConfig() + + listenAndServe, err := server.GetListenAndServeFunc( + server.WithLogger(logger), + server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), + server.WithKubeConfig(config), + server.WithDebug(*debug), + ) if err != nil { logger.Fatalf("Error setting up health/metric/pprof service: %v", err) } @@ -134,12 +145,6 @@ func main() { } }() - mgr, err := Manager(ctx, *debug) - if err != nil { - logger.WithError(err).Fatal("error configuring controller manager") - } - config := mgr.GetConfig() - // create a config that validates we're creating objects with labels validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme()) diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go index 3d79a192e0..0586f26572 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go @@ -6,11 +6,16 @@ import ( "fmt" "net/http" "path/filepath" + "time" + "github.com/go-logr/logr" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + "sigs.k8s.io/controller-runtime/pkg/log" ) // Option applies a configuration option to the given config. @@ -43,11 +48,18 @@ func WithDebug(debug bool) Option { } } +func WithKubeConfig(config *rest.Config) Option { + return func(sc *serverConfig) { + sc.kubeConfig = config + } +} + type serverConfig struct { logger *logrus.Logger tlsCertPath *string tlsKeyPath *string clientCAPath *string + kubeConfig *rest.Config debug bool } @@ -62,6 +74,7 @@ func defaultServerConfig() serverConfig { tlsCertPath: nil, tlsKeyPath: nil, clientCAPath: nil, + kubeConfig: nil, logger: nil, debug: false, } @@ -89,13 +102,60 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { return nil, fmt.Errorf("both --tls-key and --tls-crt must be provided for TLS to be enabled") } + // Create base HTTP mux with unprotected endpoints mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) profile.RegisterHandlers(mux, profile.WithTLS(tlsEnabled || !sc.debug)) + // Set up authenticated metrics endpoint if kubeConfig is provided + sc.logger.Infof("DEBUG: Checking authentication setup - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled) + + if sc.kubeConfig != nil && tlsEnabled { + sc.logger.Info("DEBUG: Setting up authenticated metrics endpoint") + + // Create authentication filter using controller-runtime + sc.logger.Info("DEBUG: Creating authentication filter with controller-runtime") + filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{ + Timeout: 30 * time.Second, + }) + if err != nil { + sc.logger.Errorf("DEBUG: Failed to create authentication filter: %v", err) + return nil, fmt.Errorf("failed to create authentication filter: %w", err) + } + sc.logger.Info("DEBUG: Authentication filter created successfully") + + // Create authenticated metrics handler + sc.logger.Info("DEBUG: Wrapping metrics handler with authentication") + logger := log.FromContext(context.Background()) + authenticatedMetricsHandler, err := filter(logger, promhttp.Handler()) + if err != nil { + sc.logger.Errorf("DEBUG: Failed to wrap metrics handler: %v", err) + return nil, fmt.Errorf("failed to wrap metrics handler with authentication: %w", err) + } + sc.logger.Info("DEBUG: Metrics handler wrapped successfully") + + // Add debugging wrapper to log authentication attempts + debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s", + r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent")) + authenticatedMetricsHandler.ServeHTTP(w, r) + }) + + mux.Handle("/metrics", debugAuthHandler) + sc.logger.Info("Metrics endpoint configured with authentication and authorization") + } else { + // Fallback to unprotected metrics (for development/testing) + sc.logger.Warnf("DEBUG: Using unprotected metrics - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled) + mux.Handle("/metrics", promhttp.Handler()) + if sc.kubeConfig == nil { + sc.logger.Warn("No Kubernetes config provided - metrics endpoint will be unprotected") + } else if !tlsEnabled { + sc.logger.Warn("TLS not enabled - metrics endpoint will be unprotected") + } + } + s := http.Server{ Handler: mux, Addr: sc.getAddress(tlsEnabled), @@ -116,6 +176,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { return nil, fmt.Errorf("error creating cert file watcher: %v", err) } csw.Run(context.Background()) + certPoolStore, err := filemonitor.NewCertPoolStore(*sc.clientCAPath) if err != nil { return nil, fmt.Errorf("certificate monitoring for client-ca failed: %v", err) @@ -141,6 +202,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { ClientAuth: tls.VerifyClientCertIfGiven, }, nil }, + NextProtos: []string{"http/1.1"}, // Disable HTTP/2 for security } return func() error { return s.ListenAndServeTLS("", "") diff --git a/vendor/modules.txt b/vendor/modules.txt index 42acf1cd02..11aaa3a006 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2275,6 +2275,7 @@ sigs.k8s.io/controller-runtime/pkg/log/zap sigs.k8s.io/controller-runtime/pkg/manager sigs.k8s.io/controller-runtime/pkg/manager/signals sigs.k8s.io/controller-runtime/pkg/metrics +sigs.k8s.io/controller-runtime/pkg/metrics/filters sigs.k8s.io/controller-runtime/pkg/metrics/server sigs.k8s.io/controller-runtime/pkg/predicate sigs.k8s.io/controller-runtime/pkg/reconcile diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go b/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go new file mode 100644 index 0000000000..1659502bcf --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go @@ -0,0 +1,122 @@ +package filters + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" + + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// WithAuthenticationAndAuthorization provides a metrics.Filter for authentication and authorization. +// Metrics will be authenticated (via TokenReviews) and authorized (via SubjectAccessReviews) with the +// kube-apiserver. +// For the authentication and authorization the controller needs a ClusterRole +// with the following rules: +// * apiGroups: authentication.k8s.io, resources: tokenreviews, verbs: create +// * apiGroups: authorization.k8s.io, resources: subjectaccessreviews, verbs: create +// +// To scrape metrics e.g. via Prometheus the client needs a ClusterRole +// with the following rule: +// * nonResourceURLs: "/metrics", verbs: get +// +// Note: Please note that configuring this metrics provider will introduce a dependency to "k8s.io/apiserver" +// to your go module. +func WithAuthenticationAndAuthorization(config *rest.Config, httpClient *http.Client) (metricsserver.Filter, error) { + authenticationV1Client, err := authenticationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + authorizationV1Client, err := authorizationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + + authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ + Anonymous: &apiserver.AnonymousAuthConfig{Enabled: false}, // Require authentication. + CacheTTL: 1 * time.Minute, + TokenAccessReviewClient: authenticationV1Client, + TokenAccessReviewTimeout: 10 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthenticator, _, err := authenticatorConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authenticator: %w", err) + } + + authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ + SubjectAccessReviewClient: authorizationV1Client, + AllowCacheTTL: 5 * time.Minute, + DenyCacheTTL: 30 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthorizer, err := authorizerConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + return func(log logr.Logger, handler http.Handler) (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + res, ok, err := delegatingAuthenticator.AuthenticateRequest(req) + if err != nil { + log.Error(err, "Authentication failed") + http.Error(w, "Authentication failed", http.StatusInternalServerError) + return + } + if !ok { + log.V(4).Info("Authentication failed") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + attributes := authorizer.AttributesRecord{ + User: res.User, + Verb: strings.ToLower(req.Method), + Path: req.URL.Path, + } + + authorized, reason, err := delegatingAuthorizer.Authorize(ctx, attributes) + if err != nil { + msg := fmt.Sprintf("Authorization for user %s failed", attributes.User.GetName()) + log.Error(err, msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + if authorized != authorizer.DecisionAllow { + msg := fmt.Sprintf("Authorization denied for user %s", attributes.User.GetName()) + log.V(4).Info(fmt.Sprintf("%s: %s", msg, reason)) + http.Error(w, msg, http.StatusForbidden) + return + } + + handler.ServeHTTP(w, req) + }), nil + }, nil +} From 7c09b002d67d40410969087213b1b7ba4410f0ed Mon Sep 17 00:00:00 2001 From: Anik Bhattacharjee Date: Tue, 23 Sep 2025 09:58:55 -0400 Subject: [PATCH 5/5] fix build error --- .../operator-lifecycle-manager/pkg/lib/server/server.go | 9 ++++----- .../operator-lifecycle-manager/pkg/lib/server/server.go | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/staging/operator-lifecycle-manager/pkg/lib/server/server.go b/staging/operator-lifecycle-manager/pkg/lib/server/server.go index 0586f26572..1be4b41091 100644 --- a/staging/operator-lifecycle-manager/pkg/lib/server/server.go +++ b/staging/operator-lifecycle-manager/pkg/lib/server/server.go @@ -8,14 +8,13 @@ import ( "path/filepath" "time" - "github.com/go-logr/logr" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" ) // Option applies a configuration option to the given config. @@ -111,10 +110,10 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { // Set up authenticated metrics endpoint if kubeConfig is provided sc.logger.Infof("DEBUG: Checking authentication setup - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled) - + if sc.kubeConfig != nil && tlsEnabled { sc.logger.Info("DEBUG: Setting up authenticated metrics endpoint") - + // Create authentication filter using controller-runtime sc.logger.Info("DEBUG: Creating authentication filter with controller-runtime") filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{ @@ -138,7 +137,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { // Add debugging wrapper to log authentication attempts debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s", + sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s", r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent")) authenticatedMetricsHandler.ServeHTTP(w, r) }) diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go index 0586f26572..1be4b41091 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/lib/server/server.go @@ -8,14 +8,13 @@ import ( "path/filepath" "time" - "github.com/go-logr/logr" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" ) // Option applies a configuration option to the given config. @@ -111,10 +110,10 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { // Set up authenticated metrics endpoint if kubeConfig is provided sc.logger.Infof("DEBUG: Checking authentication setup - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled) - + if sc.kubeConfig != nil && tlsEnabled { sc.logger.Info("DEBUG: Setting up authenticated metrics endpoint") - + // Create authentication filter using controller-runtime sc.logger.Info("DEBUG: Creating authentication filter with controller-runtime") filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{ @@ -138,7 +137,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { // Add debugging wrapper to log authentication attempts debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s", + sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s", r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent")) authenticatedMetricsHandler.ServeHTTP(w, r) })