fix: improve webhooks interoperability

flashbots · Apr 26, 2024 · 269cf88 · 269cf88
1 parent d8a1a79
commit 269cf88
Show file tree

Hide file tree

Showing 8 changed files with 223 additions and 86 deletions.
diff --git a/Makefile b/Makefile
@@ -26,4 +26,5 @@ deploy:
 		apply \
 			--filename deploy/cluster-role.yaml \
 			--filename deploy/dummy.yaml \
-			--filename deploy/deployment.yaml
+			--filename deploy/deployment-fargate.yaml \
+			--filename deploy/deployment-node-exporter.yaml
diff --git a/deploy/cluster-role.yaml b/deploy/cluster-role.yaml
@@ -1,5 +1,14 @@
 ---
 
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: kube-sidecar-injector
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector
+
+---
+
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -10,3 +19,20 @@ rules:
 - apiGroups: ["admissionregistration.k8s.io"]
   resources: ["mutatingwebhookconfigurations"]
   verbs: ["create", "get", "delete", "list", "patch", "update", "watch"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-sidecar-injector
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-sidecar-injector
+subjects:
+  - kind: ServiceAccount
+    name: kube-sidecar-injector
+    namespace: default
diff --git a/deploy/deployment-fargate.yaml b/deploy/deployment-fargate.yaml
@@ -0,0 +1,75 @@
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-sidecar-injector-fargate
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector-fargate
+spec:
+  selector:
+    app.kubernetes.io/name: kube-sidecar-injector-fargate
+  ports:
+    - name: https
+      port: 8443
+      targetPort: 8443
+
+---
+
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: kube-sidecar-injector-fargate
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector-fargate
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-sidecar-injector-fargate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-sidecar-injector-fargate
+    spec:
+      serviceAccountName: kube-sidecar-injector
+      containers:
+        - name: kube-sidecar-injector-fargate
+          image: kube-sidecar-injector:0.0.4-dev
+          args: [
+            "serve",
+            "--mutating-webhook-configuration-name", "kube-sidecar-injector-fargate",
+            "--service-name", "kube-sidecar-injector-fargate",
+          ]
+          ports:
+            - name: https
+              containerPort: 8443
+          volumeMounts:
+            - name: config
+              mountPath: /etc/kube-sidecar-injector
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: kube-sidecar-injector-fargate
+            items:
+              - key: config.yaml
+                path: config.yaml
+
+---
+
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-sidecar-injector-fargate
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector-fargate
+data:
+  config.yaml: |-
+    inject:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/name: dummy-injected-via-deployment
+
+        labels:
+          eks.amazonaws.com/fargate-profile: default
diff --git a/deploy/deployment.yaml → deploy/deployment-node-exporter.yaml b/deploy/deployment.yaml → deploy/deployment-node-exporter.yaml
@@ -1,25 +1,46 @@
 ---
 
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-sidecar-injector-node-exporter
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector-node-exporter
+spec:
+  selector:
+    app.kubernetes.io/name: kube-sidecar-injector-node-exporter
+  ports:
+    - name: https
+      port: 8443
+      targetPort: 8443
+
+---
+
 kind: Deployment
 apiVersion: apps/v1
 metadata:
-  name: kube-sidecar-injector
+  name: kube-sidecar-injector-node-exporter
   labels:
-    app.kubernetes.io/name: kube-sidecar-injector
+    app.kubernetes.io/name: kube-sidecar-injector-node-exporter
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: kube-sidecar-injector
+      app.kubernetes.io/name: kube-sidecar-injector-node-exporter
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: kube-sidecar-injector
+        app.kubernetes.io/name: kube-sidecar-injector-node-exporter
     spec:
       serviceAccountName: kube-sidecar-injector
       containers:
-        - name: kube-sidecar-injector
-          image: kube-sidecar-injector:0.0.3-dev
+        - name: kube-sidecar-injector-node-exporter
+          image: kube-sidecar-injector:0.0.4-dev
+          args: [
+            "serve",
+            "--mutating-webhook-configuration-name", "kube-sidecar-injector-node-exporter",
+            "--service-name", "kube-sidecar-injector-node-exporter",
+          ]
           ports:
             - name: https
               containerPort: 8443
@@ -30,7 +51,7 @@ spec:
       volumes:
         - name: config
           configMap:
-            name: kube-sidecar-injector
+            name: kube-sidecar-injector-node-exporter
             items:
               - key: config.yaml
                 path: config.yaml
@@ -40,11 +61,18 @@ spec:
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: kube-sidecar-injector
+  name: kube-sidecar-injector-node-exporter
+  labels:
+    app.kubernetes.io/name: kube-sidecar-injector-node-exporter
 data:
   config.yaml: |-
     inject:
-      - labels:
+      - labelSelector:
+          matchExpressions:
+            - key: eks.amazonaws.com/fargate-profile
+              operator: Exists
+
+        labels:
           flashbots.net/fargate-node-exporter: true
 
         containers:
@@ -61,50 +89,3 @@ data:
               requests:
                 cpu: 10m
                 memory: 64Mi
-
-        labelSelector:
-          matchExpressions:
-            - key: eks.amazonaws.com/fargate-profile
-              operator: Exists
-
----
-
-kind: Service
-apiVersion: v1
-metadata:
-  name: kube-sidecar-injector
-  labels:
-    app.kubernetes.io/name: kube-sidecar-injector
-spec:
-  selector:
-    app.kubernetes.io/name: kube-sidecar-injector
-  ports:
-    - name: https
-      port: 8443
-      targetPort: 8443
-
----
-
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: kube-sidecar-injector
-  labels:
-    app.kubernetes.io/name: kube-sidecar-injector
-
----
-
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kube-sidecar-injector
-  labels:
-    app.kubernetes.io/name: kube-sidecar-injector
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-sidecar-injector
-subjects:
-  - kind: ServiceAccount
-    name: kube-sidecar-injector
-    namespace: default
diff --git a/deploy/dummy.yaml b/deploy/dummy.yaml
@@ -30,7 +30,7 @@ metadata:
 
 spec:
   containers:
-    - name: dummy
+    - name: dummy-injected
       image: ubuntu
       command:
         - /bin/bash
@@ -78,3 +78,35 @@ spec:
           trap stop SIGTERM
           trap stop SIGINT
           while [[ ! -f stop ]]; do sleep 1; done
+
+---
+
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: dummy-injected-via-deployment
+  labels:
+    app.kubernetes.io/name: dummy-injected-via-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: dummy-injected-via-deployment
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: dummy-injected-via-deployment
+    spec:
+      containers:
+        - name: dummy-injected-via-deployment
+          image: ubuntu
+          command:
+            - /bin/bash
+            - -c
+            - |-
+              stop() {
+                touch stop
+              }
+              trap stop SIGTERM
+              trap stop SIGINT
+              while [[ ! -f stop ]]; do sleep 1; done
diff --git a/patch/update_pod_labels.go b/patch/update_pod_labels.go
@@ -14,7 +14,7 @@ func UpdatePodLabels(
 		return nil, nil
 	}
 
-	if len(pod.Annotations) == 0 {
+	if len(pod.Labels) == 0 {
 		op, err := operation.Add("/metadata/labels", labels)
 		if err != nil {
 			return nil, err
@@ -25,7 +25,7 @@ func UpdatePodLabels(
 	res := make(json_patch.Patch, 0, len(labels))
 
 	for k, v := range labels {
-		if _, exists := pod.Annotations[k]; exists {
+		if _, exists := pod.Labels[k]; exists {
 			op, err := operation.Replace("/metadata/labels/"+operation.Escape(k), v)
 			if err != nil {
 				return nil, err

diff --git a/readme.md b/readme.md
@@ -10,8 +10,13 @@ running next to it.
 
 ```yaml
 inject:
-  - labels:
-      flashbots.net/fargate-node-exporter: true
+  - labelSelector:
+      matchExpressions:
+        - key: eks.amazonaws.com/fargate-profile
+          operator: Exists
+
+    labels:
+      flashbots.net/prometheus-node-exporter: true
 
     containers:
       - name: node-exporter
@@ -28,8 +33,14 @@ inject:
             cpu: 10m
             memory: 64Mi
 
-    labelSelector:
-      matchExpressions:
-        - key: eks.amazonaws.com/fargate-profile
-          operator: Exists
 ```
+
+### Caveats
+
+Single webhook configuration can me configured to apply multiple injection
+rules.  However, if these rules are supposed to interact somehow (for example
+rule A introduces changes that rule B is supposed to act upon) then they should
+be placed into _separate_ webhooks.
+
+See k8s webhook [reinvocation policy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)
+for the details.