Merge pull request #1070 from machine424/smon

MON-669: Add etcd ServiceMonitor, before that was managed by cluster-…
openshift · Jul 17, 2023 · 5979f3b · 5979f3b
2 parents e77e5d2 + 2091dd9
commit 5979f3b
Show file tree

Hide file tree

Showing 6 changed files with 215 additions and 15 deletions.
diff --git a/bindata/etcd/minimal-sm.yaml b/bindata/etcd/minimal-sm.yaml
@@ -0,0 +1,38 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: etcd-minimal
+  namespace: openshift-etcd-operator
+  labels:
+    app.kubernetes.io/name: etcd
+    k8s-app: etcd
+    monitoring.openshift.io/collection-profile: minimal
+spec:
+  endpoints:
+    - interval: 30s
+      metricRelabelings:
+      - action: keep
+        regex: (etcd_disk_backend_commit_duration_seconds_bucket|etcd_disk_wal_fsync_duration_seconds_bucket|etcd_mvcc_db_total_size_in_bytes|etcd_mvcc_db_total_size_in_use_in_bytes|etcd_network_peer_round_trip_time_seconds_bucket|etcd_network_peer_sent_failures_total|etcd_server_has_leader|etcd_server_is_leader|etcd_server_proposals_failed_total|etcd_server_quota_backend_bytes|grpc_server_handled_total|grpc_server_handling_seconds_bucket|grpc_server_started_total|process_start_time_seconds)
+        sourceLabels:
+        - __name__
+      port: etcd-metrics
+      scheme: https
+      tlsConfig:
+        ca:
+          configMap:
+            name: etcd-metric-serving-ca
+            key: ca-bundle.crt
+        cert:
+          secret:
+            name: etcd-metric-client
+            key: tls.crt
+        keySecret:
+          name: etcd-metric-client
+          key: tls.key
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - openshift-etcd
+  selector:
+    matchLabels:
+      k8s-app: etcd
diff --git a/bindata/etcd/sm.yaml b/bindata/etcd/sm.yaml
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: etcd
+  namespace: openshift-etcd-operator
+  labels:
+    app.kubernetes.io/name: etcd
+    k8s-app: etcd
+    monitoring.openshift.io/collection-profile: full
+spec:
+  endpoints:
+    - interval: 30s
+      port: etcd-metrics
+      scheme: https
+      tlsConfig:
+        ca:
+          configMap:
+            name: etcd-metric-serving-ca
+            key: ca-bundle.crt
+        cert:
+          secret:
+            name: etcd-metric-client
+            key: tls.crt
+        keySecret:
+          name: etcd-metric-client
+          key: tls.key
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - openshift-etcd
+  selector:
+    matchLabels:
+      k8s-app: etcd
diff --git a/docs/etcd-tls-assets.md b/docs/etcd-tls-assets.md
@@ -27,14 +27,15 @@ for these certificates.
 All etcd CAs and their CA bundles are stored in the `openshift-config`
 namespace.
 
-| CA (secret)        | CA bundle (configmap)            | CA bundle also appearing in                  |
-| ------------------ | -------------------------------- | -------------------------------------------- |
-| etcd-signer        | etcd-ca-bundle                   | openshift-etcd                               |
-|                    |                                  | openshift-etcd-operator                      |
-|                    |                                  | openshift-etcd/etcd-peer-client-ca           |
-|                    | etcd-serving-ca                  | openshift-etcd                               |
-| etcd-metric-signer | etcd-metric-serving-ca           | openshift-etcd/etcd-metrics-proxy-client-ca  |
-|                    |                                  | openshift-etcd/etcd-metrics-proxy-serving-ca |
+| CA (secret)        | CA bundle (configmap)            | CA bundle also appearing in                    |
+| ------------------ | -------------------------------- | -----------------------------------------------|
+| etcd-signer        | etcd-ca-bundle                   | openshift-etcd                                 |
+|                    |                                  | openshift-etcd-operator                        |
+|                    |                                  | openshift-etcd/etcd-peer-client-ca             |
+|                    | etcd-serving-ca                  | openshift-etcd                                 |
+| etcd-metric-signer | etcd-metric-serving-ca           | openshift-etcd/etcd-metrics-proxy-client-ca    |
+|                    |                                  | openshift-etcd/etcd-metrics-proxy-serving-ca   |
+|                    |                                  | openshift-etcd-operator/etcd-metric-serving-ca |
 
 ## etcd cert summary
 
@@ -46,7 +47,7 @@ All etcd certificates are stored in secrets.
 |                    |                                           |                                  | openshift-etcd-operator                     |
 |                    | openshift-etcd/etcd-peer-$node            | etcd peer communication          | collected in etcd-all-certs                 |
 |                    | openshift-etcd/etcd-serving-$node         | etcd member serving              | collected in etcd-all-certs                 |
-| etcd-metric-signer | openshift-config/etcd-metric-client       | authn prometheus to etcd metrics | openshift-monitoring/kube-etcd-client-certs |
+| etcd-metric-signer | openshift-config/etcd-metric-client       | authn prometheus to etcd metrics | openshift-etcd-operator/etcd-metric-client  |
 |                    | openshift-etcd/etcd-serving-metrics-$node | etcd member metrics serving      | collected in etcd-all-certs                 |
 
 ## etcd-signer and etcd-metric-signer CA certs
@@ -144,10 +145,7 @@ Certificate:
 ```
 
 Similarly, the `etcd-metric-signer` CA issues a client cert that
-prometheus uses when authenticating with the etcd metrics server. The
-`cluster-monitoring-operator` copies this into the
-`kube-etcd-client-certs` secret in the `openshift-monitoring`
-namespace.
+prometheus uses when authenticating with the etcd metrics server.
 
 ```
 $ oc get -n openshift-config secret/etcd-metric-client -o template='{{index .data "tls.crt"}}'  | base64 -d | openssl x509 -noout -text

diff --git a/pkg/operator/etcd_assets/bindata.go b/pkg/operator/etcd_assets/bindata.go
diff --git a/pkg/operator/resourcesynccontroller/resourcesynccontroller.go b/pkg/operator/resourcesynccontroller/resourcesynccontroller.go
@@ -52,6 +52,13 @@ func NewResourceSyncController(
 		return nil, err
 	}
 
+	if err := resourceSyncController.SyncSecret(
+		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.OperatorNamespace, Name: "etcd-metric-client"},
+		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "etcd-metric-client"},
+	); err != nil {
+		return nil, err
+	}
+
 	if err := resourceSyncController.SyncConfigMap(
 		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "etcd-peer-client-ca"},
 		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "etcd-ca-bundle"},
@@ -64,6 +71,12 @@ func NewResourceSyncController(
 	); err != nil {
 		return nil, err
 	}
+	if err := resourceSyncController.SyncConfigMap(
+		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.OperatorNamespace, Name: "etcd-metric-serving-ca"},
+		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "etcd-metric-serving-ca"},
+	); err != nil {
+		return nil, err
+	}
 	if err := resourceSyncController.SyncConfigMap(
 		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "etcd-serving-ca"},
 		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "etcd-serving-ca"},

diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go
@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/dynamic"
 	corev1informers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
@@ -77,6 +78,10 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 	if err != nil {
 		return err
 	}
+	dynamicClient, err := dynamic.NewForConfig(controllerContext.ProtoKubeConfig)
+	if err != nil {
+		return err
+	}
 	operatorConfigClient, err := operatorversionedclient.NewForConfig(controllerContext.KubeConfig)
 	if err != nil {
 		return err
@@ -185,11 +190,13 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 			"etcd/ns.yaml",
 			"etcd/sa.yaml",
 			"etcd/svc.yaml",
+			"etcd/sm.yaml",
+			"etcd/minimal-sm.yaml",
 		},
-		(&resourceapply.ClientHolder{}).WithKubernetes(kubeClient),
+		(&resourceapply.ClientHolder{}).WithKubernetes(kubeClient).WithDynamicClient(dynamicClient),
 		operatorClient,
 		controllerContext.EventRecorder,
-	).AddKubeInformers(kubeInformersForNamespaces)
+	).WithIgnoreNotFoundOnCreate().AddKubeInformers(kubeInformersForNamespaces)
 
 	envVarController := etcdenvvar.NewEnvVarController(
 		os.Getenv("IMAGE"),